2024-11-18 03:14:36,714 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-18 03:14:36,733 main DEBUG Took 0.016404 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 03:14:36,733 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 03:14:36,734 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 03:14:36,735 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 03:14:36,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,760 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 03:14:36,781 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,786 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,787 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,789 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,790 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,792 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,792 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,793 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,794 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,795 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,796 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,797 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:14:36,798 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,799 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 03:14:36,801 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:14:36,803 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 03:14:36,806 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 03:14:36,807 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 03:14:36,808 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 03:14:36,809 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 03:14:36,820 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 03:14:36,824 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 03:14:36,826 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 03:14:36,827 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 03:14:36,827 main DEBUG createAppenders(={Console}) 2024-11-18 03:14:36,829 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-18 03:14:36,829 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-18 03:14:36,830 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-18 03:14:36,832 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 03:14:36,832 main DEBUG OutputStream closed 2024-11-18 03:14:36,833 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 03:14:36,833 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 03:14:36,833 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-18 03:14:36,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 03:14:36,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 03:14:36,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 03:14:36,937 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 03:14:36,938 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 03:14:36,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 03:14:36,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 03:14:36,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 03:14:36,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 03:14:36,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 03:14:36,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 03:14:36,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 03:14:36,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 03:14:36,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 03:14:36,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 03:14:36,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 03:14:36,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 03:14:36,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 03:14:36,947 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 03:14:36,948 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-18 03:14:36,948 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 03:14:36,949 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-18T03:14:37,316 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5 2024-11-18 03:14:37,321 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 03:14:37,321 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T03:14:37,334 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-18T03:14:37,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T03:14:37,411 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7, deleteOnExit=true 2024-11-18T03:14:37,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T03:14:37,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/test.cache.data in system properties and HBase conf 2024-11-18T03:14:37,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T03:14:37,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/hadoop.log.dir in system properties and HBase conf 2024-11-18T03:14:37,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T03:14:37,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T03:14:37,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T03:14:37,539 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T03:14:37,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:14:37,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T03:14:37,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T03:14:37,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T03:14:37,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T03:14:37,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T03:14:37,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T03:14:37,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T03:14:37,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T03:14:37,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T03:14:37,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/nfs.dump.dir in system properties and HBase conf 2024-11-18T03:14:37,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/java.io.tmpdir in system properties and HBase conf 2024-11-18T03:14:37,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T03:14:37,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T03:14:37,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T03:14:39,150 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T03:14:39,237 INFO [Time-limited test {}] log.Log(170): Logging initialized @3473ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T03:14:39,328 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T03:14:39,410 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T03:14:39,488 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T03:14:39,489 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T03:14:39,492 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T03:14:39,514 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T03:14:39,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/hadoop.log.dir/,AVAILABLE} 2024-11-18T03:14:39,523 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T03:14:39,803 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f7463d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/java.io.tmpdir/jetty-localhost-42499-hadoop-hdfs-3_4_1-tests_jar-_-any-9117732295256627783/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T03:14:39,814 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:42499} 2024-11-18T03:14:39,815 INFO [Time-limited test {}] server.Server(415): Started @4052ms 2024-11-18T03:14:40,514 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T03:14:40,523 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T03:14:40,524 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T03:14:40,525 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T03:14:40,525 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T03:14:40,525 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/hadoop.log.dir/,AVAILABLE} 2024-11-18T03:14:40,526 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T03:14:40,653 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70061d7e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/java.io.tmpdir/jetty-localhost-37175-hadoop-hdfs-3_4_1-tests_jar-_-any-1287625048549691874/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T03:14:40,654 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:37175} 2024-11-18T03:14:40,655 INFO [Time-limited test {}] server.Server(415): Started @4892ms 2024-11-18T03:14:40,722 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T03:14:42,052 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/data/data2/current/BP-39717385-172.17.0.3-1731899678439/current, will proceed with Du for space computation calculation, 2024-11-18T03:14:42,052 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/data/data1/current/BP-39717385-172.17.0.3-1731899678439/current, will proceed with Du for space computation calculation, 2024-11-18T03:14:42,106 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T03:14:42,170 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x413e726ab279ca7f with lease ID 0xa028b71618e7be19: Processing first storage report for DS-2234658c-9969-4913-a03c-1f3d9e90d792 from datanode DatanodeRegistration(127.0.0.1:44343, datanodeUuid=58cfbc07-69ae-4c4e-8282-2160a7242b4a, infoPort=39963, infoSecurePort=0, ipcPort=45347, storageInfo=lv=-57;cid=testClusterID;nsid=1719656249;c=1731899678439) 2024-11-18T03:14:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x413e726ab279ca7f with lease ID 0xa028b71618e7be19: from storage DS-2234658c-9969-4913-a03c-1f3d9e90d792 node DatanodeRegistration(127.0.0.1:44343, datanodeUuid=58cfbc07-69ae-4c4e-8282-2160a7242b4a, infoPort=39963, infoSecurePort=0, ipcPort=45347, storageInfo=lv=-57;cid=testClusterID;nsid=1719656249;c=1731899678439), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T03:14:42,174 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x413e726ab279ca7f with lease ID 0xa028b71618e7be19: Processing first storage report for DS-f8ce3371-80eb-411f-83ac-09bc0dbf48de from datanode DatanodeRegistration(127.0.0.1:44343, datanodeUuid=58cfbc07-69ae-4c4e-8282-2160a7242b4a, infoPort=39963, infoSecurePort=0, ipcPort=45347, storageInfo=lv=-57;cid=testClusterID;nsid=1719656249;c=1731899678439) 2024-11-18T03:14:42,174 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x413e726ab279ca7f with lease ID 0xa028b71618e7be19: from storage DS-f8ce3371-80eb-411f-83ac-09bc0dbf48de node DatanodeRegistration(127.0.0.1:44343, datanodeUuid=58cfbc07-69ae-4c4e-8282-2160a7242b4a, infoPort=39963, infoSecurePort=0, ipcPort=45347, storageInfo=lv=-57;cid=testClusterID;nsid=1719656249;c=1731899678439), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T03:14:42,220 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5 2024-11-18T03:14:42,381 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/zookeeper_0, clientPort=56749, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T03:14:42,392 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56749 2024-11-18T03:14:42,413 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T03:14:42,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T03:14:42,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741825_1001 (size=7) 2024-11-18T03:14:43,113 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 with version=8 2024-11-18T03:14:43,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/hbase-staging 2024-11-18T03:14:43,220 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T03:14:43,538 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fd862c3a04f9:0 server-side Connection retries=45 2024-11-18T03:14:43,552 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:14:43,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:14:43,561 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:14:43,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:14:43,562 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:14:43,776 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T03:14:43,880 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T03:14:43,892 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T03:14:43,897 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:14:43,936 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3556 (auto-detected) 2024-11-18T03:14:43,937 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-18T03:14:43,964 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45051 2024-11-18T03:14:44,005 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45051 connecting to ZooKeeper ensemble=127.0.0.1:56749 2024-11-18T03:14:44,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450510x0, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:14:44,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45051-0x1014befe9de0000 connected 2024-11-18T03:14:44,264 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T03:14:44,267 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T03:14:44,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:14:44,285 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1, hbase.cluster.distributed=false 2024-11-18T03:14:44,324 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:14:44,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45051 2024-11-18T03:14:44,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45051 2024-11-18T03:14:44,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45051 2024-11-18T03:14:44,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45051 2024-11-18T03:14:44,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45051 2024-11-18T03:14:44,517 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fd862c3a04f9:0 server-side Connection retries=45 2024-11-18T03:14:44,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:14:44,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:14:44,519 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:14:44,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:14:44,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:14:44,522 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T03:14:44,525 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:14:44,526 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38549 2024-11-18T03:14:44,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38549 connecting to ZooKeeper ensemble=127.0.0.1:56749 2024-11-18T03:14:44,531 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T03:14:44,535 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T03:14:44,549 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385490x0, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:14:44,547 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385490x0, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:14:44,550 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38549-0x1014befe9de0001 connected 2024-11-18T03:14:44,554 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T03:14:44,567 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T03:14:44,573 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T03:14:44,580 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:14:44,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38549 2024-11-18T03:14:44,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38549 2024-11-18T03:14:44,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38549 2024-11-18T03:14:44,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38549 2024-11-18T03:14:44,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38549 2024-11-18T03:14:44,598 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fd862c3a04f9:45051 2024-11-18T03:14:44,599 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:44,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:14:44,611 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:14:44,613 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:44,642 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T03:14:44,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:44,642 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:44,643 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T03:14:44,644 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fd862c3a04f9,45051,1731899683325 from backup master directory 2024-11-18T03:14:44,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:44,652 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:14:44,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:14:44,654 WARN [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:14:44,654 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:44,656 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T03:14:44,658 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T03:14:44,720 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/hbase.id] with ID: d21d32cf-9d00-4e6e-954c-ada723a6fe6a 2024-11-18T03:14:44,720 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/.tmp/hbase.id 2024-11-18T03:14:44,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741826_1002 (size=42) 2024-11-18T03:14:45,133 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/.tmp/hbase.id]:[hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/hbase.id] 2024-11-18T03:14:45,184 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T03:14:45,191 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T03:14:45,214 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-18T03:14:45,263 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:45,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:45,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741827_1003 (size=196) 2024-11-18T03:14:45,702 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:14:45,705 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T03:14:45,723 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:45,729 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:14:45,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741828_1004 (size=1189) 2024-11-18T03:14:46,176 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store 2024-11-18T03:14:46,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741829_1005 (size=34) 2024-11-18T03:14:46,602 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T03:14:46,606 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:14:46,607 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:14:46,607 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:14:46,607 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:14:46,609 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:14:46,609 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:14:46,610 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:14:46,611 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731899686607Disabling compacts and flushes for region at 1731899686607Disabling writes for close at 1731899686609 (+2 ms)Writing region close event to WAL at 1731899686610 (+1 ms)Closed at 1731899686610 2024-11-18T03:14:46,613 WARN [master/fd862c3a04f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/.initializing 2024-11-18T03:14:46,614 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/WALs/fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:46,621 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:14:46,635 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fd862c3a04f9%2C45051%2C1731899683325, suffix=, logDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/WALs/fd862c3a04f9,45051,1731899683325, archiveDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/oldWALs, maxLogs=10 2024-11-18T03:14:46,655 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/WALs/fd862c3a04f9,45051,1731899683325/fd862c3a04f9%2C45051%2C1731899683325.1731899686639, exclude list is [], retry=0 2024-11-18T03:14:46,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44343,DS-2234658c-9969-4913-a03c-1f3d9e90d792,DISK] 2024-11-18T03:14:46,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T03:14:46,703 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/WALs/fd862c3a04f9,45051,1731899683325/fd862c3a04f9%2C45051%2C1731899683325.1731899686639 2024-11-18T03:14:46,704 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39963:39963)] 2024-11-18T03:14:46,704 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:14:46,705 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:14:46,708 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,709 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T03:14:46,768 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:46,770 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:46,771 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,774 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T03:14:46,774 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:46,775 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:14:46,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,778 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T03:14:46,779 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:46,780 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:14:46,780 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T03:14:46,783 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:46,784 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:14:46,784 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,789 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,790 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,795 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,796 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,799 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:14:46,804 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:14:46,808 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:14:46,809 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71791364, jitterRate=0.06977468729019165}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:14:46,815 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731899686720Initializing all the Stores at 1731899686722 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731899686722Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899686723 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899686723Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899686723Cleaning up temporary data from old regions at 1731899686796 (+73 ms)Region opened successfully at 1731899686815 (+19 ms) 2024-11-18T03:14:46,817 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T03:14:46,848 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521cbb37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fd862c3a04f9/172.17.0.3:0 2024-11-18T03:14:46,876 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T03:14:46,886 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T03:14:46,886 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T03:14:46,889 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T03:14:46,890 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T03:14:46,895 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T03:14:46,895 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T03:14:46,919 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T03:14:46,927 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T03:14:46,978 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T03:14:46,981 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T03:14:46,983 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T03:14:46,989 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T03:14:46,991 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T03:14:46,994 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T03:14:46,999 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T03:14:47,001 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T03:14:47,010 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T03:14:47,028 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T03:14:47,031 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T03:14:47,041 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:14:47,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:14:47,042 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:47,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:47,045 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fd862c3a04f9,45051,1731899683325, sessionid=0x1014befe9de0000, setting cluster-up flag (Was=false) 2024-11-18T03:14:47,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:47,073 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:47,105 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T03:14:47,107 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:47,126 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:47,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:47,157 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T03:14:47,159 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:47,165 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T03:14:47,189 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(746): ClusterId : d21d32cf-9d00-4e6e-954c-ada723a6fe6a 2024-11-18T03:14:47,192 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T03:14:47,202 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T03:14:47,203 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T03:14:47,211 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T03:14:47,212 DEBUG [RS:0;fd862c3a04f9:38549 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391500b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fd862c3a04f9/172.17.0.3:0 2024-11-18T03:14:47,226 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fd862c3a04f9:38549 2024-11-18T03:14:47,229 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T03:14:47,229 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T03:14:47,229 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T03:14:47,232 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(2659): reportForDuty to master=fd862c3a04f9,45051,1731899683325 with port=38549, startcode=1731899684479 2024-11-18T03:14:47,243 DEBUG [RS:0;fd862c3a04f9:38549 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T03:14:47,245 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T03:14:47,254 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T03:14:47,262 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T03:14:47,267 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fd862c3a04f9,45051,1731899683325 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T03:14:47,275 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fd862c3a04f9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:14:47,274 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53373, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T03:14:47,275 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fd862c3a04f9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:14:47,275 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fd862c3a04f9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:14:47,275 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fd862c3a04f9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:14:47,276 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fd862c3a04f9:0, corePoolSize=10, maxPoolSize=10 2024-11-18T03:14:47,276 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,276 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fd862c3a04f9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:14:47,276 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,279 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731899717279 2024-11-18T03:14:47,280 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:47,281 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T03:14:47,282 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:14:47,282 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T03:14:47,283 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T03:14:47,287 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T03:14:47,287 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T03:14:47,287 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T03:14:47,288 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T03:14:47,289 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:47,290 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T03:14:47,289 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,292 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T03:14:47,293 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T03:14:47,293 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T03:14:47,297 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T03:14:47,297 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T03:14:47,299 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fd862c3a04f9:0:becomeActiveMaster-HFileCleaner.large.0-1731899687298,5,FailOnTimeoutGroup] 2024-11-18T03:14:47,300 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fd862c3a04f9:0:becomeActiveMaster-HFileCleaner.small.0-1731899687299,5,FailOnTimeoutGroup] 2024-11-18T03:14:47,300 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,300 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T03:14:47,301 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,302 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741831_1007 (size=1321) 2024-11-18T03:14:47,309 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T03:14:47,309 WARN [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T03:14:47,410 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(2659): reportForDuty to master=fd862c3a04f9,45051,1731899683325 with port=38549, startcode=1731899684479 2024-11-18T03:14:47,413 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45051 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:47,416 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45051 {}] master.ServerManager(517): Registering regionserver=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:47,426 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:14:47,427 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35537 2024-11-18T03:14:47,427 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T03:14:47,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:14:47,443 DEBUG [RS:0;fd862c3a04f9:38549 {}] zookeeper.ZKUtil(111): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:47,443 WARN [RS:0;fd862c3a04f9:38549 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:14:47,443 INFO [RS:0;fd862c3a04f9:38549 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:14:47,443 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/WALs/fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:47,446 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fd862c3a04f9,38549,1731899684479] 2024-11-18T03:14:47,470 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T03:14:47,486 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T03:14:47,491 INFO [RS:0;fd862c3a04f9:38549 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T03:14:47,491 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,492 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T03:14:47,498 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T03:14:47,499 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,499 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,500 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,500 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,500 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,500 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,500 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fd862c3a04f9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:14:47,500 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,501 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,501 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,501 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,502 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,502 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fd862c3a04f9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:14:47,502 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fd862c3a04f9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:14:47,502 DEBUG [RS:0;fd862c3a04f9:38549 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:14:47,503 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,503 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,503 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,504 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,504 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,504 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,38549,1731899684479-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:14:47,524 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T03:14:47,526 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,38549,1731899684479-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,526 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,526 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.Replication(171): fd862c3a04f9,38549,1731899684479 started 2024-11-18T03:14:47,543 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:47,543 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1482): Serving as fd862c3a04f9,38549,1731899684479, RpcServer on fd862c3a04f9/172.17.0.3:38549, sessionid=0x1014befe9de0001 2024-11-18T03:14:47,544 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T03:14:47,544 DEBUG [RS:0;fd862c3a04f9:38549 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:47,544 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fd862c3a04f9,38549,1731899684479' 2024-11-18T03:14:47,544 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T03:14:47,545 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T03:14:47,546 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T03:14:47,546 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T03:14:47,546 DEBUG [RS:0;fd862c3a04f9:38549 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:47,546 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fd862c3a04f9,38549,1731899684479' 2024-11-18T03:14:47,546 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T03:14:47,547 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T03:14:47,548 DEBUG [RS:0;fd862c3a04f9:38549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T03:14:47,548 INFO [RS:0;fd862c3a04f9:38549 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T03:14:47,548 INFO [RS:0;fd862c3a04f9:38549 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T03:14:47,652 INFO [RS:0;fd862c3a04f9:38549 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:14:47,668 INFO [RS:0;fd862c3a04f9:38549 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fd862c3a04f9%2C38549%2C1731899684479, suffix=, logDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/WALs/fd862c3a04f9,38549,1731899684479, archiveDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/oldWALs, maxLogs=32 2024-11-18T03:14:47,688 DEBUG [RS:0;fd862c3a04f9:38549 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/WALs/fd862c3a04f9,38549,1731899684479/fd862c3a04f9%2C38549%2C1731899684479.1731899687673, exclude list is [], retry=0 2024-11-18T03:14:47,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44343,DS-2234658c-9969-4913-a03c-1f3d9e90d792,DISK] 2024-11-18T03:14:47,697 INFO [RS:0;fd862c3a04f9:38549 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/WALs/fd862c3a04f9,38549,1731899684479/fd862c3a04f9%2C38549%2C1731899684479.1731899687673 2024-11-18T03:14:47,697 DEBUG [RS:0;fd862c3a04f9:38549 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39963:39963)] 2024-11-18T03:14:47,704 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T03:14:47,705 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:14:47,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741833_1009 (size=32) 2024-11-18T03:14:48,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:14:48,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T03:14:48,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:14:48,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T03:14:48,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:14:48,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T03:14:48,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:14:48,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T03:14:48,137 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:14:48,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:14:48,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740 2024-11-18T03:14:48,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740 2024-11-18T03:14:48,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:14:48,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:14:48,146 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:14:48,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:14:48,153 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:14:48,155 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62799045, jitterRate=-0.06422130763530731}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:14:48,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731899688118Initializing all the Stores at 1731899688120 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731899688120Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731899688121 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899688121Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731899688121Cleaning up temporary data from old regions at 1731899688145 (+24 ms)Region opened successfully at 1731899688158 (+13 ms) 2024-11-18T03:14:48,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:14:48,158 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:14:48,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:14:48,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:14:48,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:14:48,160 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:14:48,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731899688158Disabling compacts and flushes for region at 1731899688158Disabling writes for close at 1731899688159 (+1 ms)Writing region close event to WAL at 1731899688160 (+1 ms)Closed at 1731899688160 2024-11-18T03:14:48,164 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:14:48,164 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T03:14:48,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T03:14:48,176 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:14:48,179 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T03:14:48,335 DEBUG [fd862c3a04f9:45051 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T03:14:48,351 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:48,355 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fd862c3a04f9,38549,1731899684479, state=OPENING 2024-11-18T03:14:48,410 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T03:14:48,421 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:48,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:14:48,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:14:48,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:14:48,424 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:14:48,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:14:48,607 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T03:14:48,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54867, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T03:14:48,622 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T03:14:48,623 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:14:48,623 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T03:14:48,626 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fd862c3a04f9%2C38549%2C1731899684479.meta, suffix=.meta, logDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/WALs/fd862c3a04f9,38549,1731899684479, archiveDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/oldWALs, maxLogs=32 2024-11-18T03:14:48,641 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/WALs/fd862c3a04f9,38549,1731899684479/fd862c3a04f9%2C38549%2C1731899684479.meta.1731899688628.meta, exclude list is [], retry=0 2024-11-18T03:14:48,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44343,DS-2234658c-9969-4913-a03c-1f3d9e90d792,DISK] 2024-11-18T03:14:48,648 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/WALs/fd862c3a04f9,38549,1731899684479/fd862c3a04f9%2C38549%2C1731899684479.meta.1731899688628.meta 2024-11-18T03:14:48,648 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39963:39963)] 2024-11-18T03:14:48,649 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:14:48,650 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T03:14:48,653 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T03:14:48,657 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T03:14:48,661 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T03:14:48,662 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:14:48,662 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T03:14:48,662 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T03:14:48,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T03:14:48,667 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:14:48,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T03:14:48,670 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:14:48,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T03:14:48,672 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:14:48,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T03:14:48,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:14:48,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:48,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:14:48,677 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:14:48,678 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740 2024-11-18T03:14:48,681 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740 2024-11-18T03:14:48,683 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:14:48,683 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:14:48,684 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:14:48,687 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:14:48,689 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61423747, jitterRate=-0.0847148448228836}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:14:48,689 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T03:14:48,691 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731899688663Writing region info on filesystem at 1731899688663Initializing all the Stores at 1731899688665 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731899688665Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731899688665Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899688665Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731899688665Cleaning up temporary data from old regions at 1731899688684 (+19 ms)Running coprocessor post-open hooks at 1731899688689 (+5 ms)Region opened successfully at 1731899688691 (+2 ms) 2024-11-18T03:14:48,697 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731899688598 2024-11-18T03:14:48,708 DEBUG [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T03:14:48,708 INFO [RS_OPEN_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T03:14:48,711 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:48,715 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fd862c3a04f9,38549,1731899684479, state=OPEN 2024-11-18T03:14:48,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:14:48,765 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:14:48,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:14:48,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:14:48,765 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:48,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T03:14:48,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fd862c3a04f9,38549,1731899684479 in 339 msec 2024-11-18T03:14:48,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T03:14:48,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 603 msec 2024-11-18T03:14:48,779 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:14:48,779 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T03:14:48,796 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:48,797 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:48,814 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:48,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50031, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:48,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6400 sec 2024-11-18T03:14:48,835 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731899688835, completionTime=-1 2024-11-18T03:14:48,837 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T03:14:48,838 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T03:14:48,860 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-18T03:14:48,861 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731899748860 2024-11-18T03:14:48,861 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731899808861 2024-11-18T03:14:48,861 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-18T03:14:48,863 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,45051,1731899683325-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:48,863 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,45051,1731899683325-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:48,863 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,45051,1731899683325-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:48,864 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fd862c3a04f9:45051, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:48,865 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:48,867 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:48,871 DEBUG [master/fd862c3a04f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T03:14:48,894 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.240sec 2024-11-18T03:14:48,895 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T03:14:48,896 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T03:14:48,897 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T03:14:48,897 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T03:14:48,898 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T03:14:48,898 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,45051,1731899683325-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:14:48,899 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,45051,1731899683325-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T03:14:48,907 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T03:14:48,908 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T03:14:48,909 INFO [master/fd862c3a04f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fd862c3a04f9,45051,1731899683325-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:14:49,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e13c3ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:49,004 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T03:14:49,004 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T03:14:49,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:49,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:49,015 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e3743c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:49,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:49,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361bc9ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:49,028 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:49,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:49,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:49,068 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=916, ProcessCount=11, AvailableMemoryMB=2900 2024-11-18T03:14:49,076 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:14:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57db5d72 2024-11-18T03:14:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:14:49,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:14:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:14:49,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:14:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T03:14:49,108 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:14:49,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-18T03:14:49,110 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:49,113 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:14:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:14:49,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741835_1011 (size=1061) 2024-11-18T03:14:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:14:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:14:49,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:14:49,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741836_1012 (size=53) 2024-11-18T03:14:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:14:49,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:14:49,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 29141263847cbdef9bad2179de1271a3, disabling compactions & flushes 2024-11-18T03:14:49,941 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:49,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:49,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. after waiting 0 ms 2024-11-18T03:14:49,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:49,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:49,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29141263847cbdef9bad2179de1271a3: Waiting for close lock at 1731899689941Disabling compacts and flushes for region at 1731899689941Disabling writes for close at 1731899689942 (+1 ms)Writing region close event to WAL at 1731899689942Closed at 1731899689942 2024-11-18T03:14:49,946 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:14:49,954 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731899689946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731899689946"}]},"ts":"1731899689946"} 2024-11-18T03:14:49,958 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:14:49,960 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:14:49,963 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899689960"}]},"ts":"1731899689960"} 2024-11-18T03:14:49,968 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T03:14:49,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, ASSIGN}] 2024-11-18T03:14:49,972 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, ASSIGN 2024-11-18T03:14:49,974 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, ASSIGN; state=OFFLINE, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=false 2024-11-18T03:14:50,127 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=29141263847cbdef9bad2179de1271a3, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:50,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, ASSIGN because future has completed 2024-11-18T03:14:50,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:14:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:14:50,299 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:50,300 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:14:50,300 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,300 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:14:50,301 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,301 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,304 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,309 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:14:50,309 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29141263847cbdef9bad2179de1271a3 columnFamilyName A 2024-11-18T03:14:50,309 DEBUG [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:50,311 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.HStore(327): Store=29141263847cbdef9bad2179de1271a3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:14:50,311 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,314 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:14:50,315 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29141263847cbdef9bad2179de1271a3 columnFamilyName B 2024-11-18T03:14:50,315 DEBUG [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:50,316 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.HStore(327): Store=29141263847cbdef9bad2179de1271a3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:14:50,316 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,318 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:14:50,318 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29141263847cbdef9bad2179de1271a3 columnFamilyName C 2024-11-18T03:14:50,318 DEBUG [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:14:50,319 INFO [StoreOpener-29141263847cbdef9bad2179de1271a3-1 {}] regionserver.HStore(327): Store=29141263847cbdef9bad2179de1271a3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:14:50,320 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:50,320 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,322 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,323 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,325 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,325 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,326 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:14:50,328 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,332 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:14:50,333 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 29141263847cbdef9bad2179de1271a3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73103739, jitterRate=0.08933059871196747}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:14:50,333 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:50,334 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 29141263847cbdef9bad2179de1271a3: Running coprocessor pre-open hook at 1731899690301Writing region info on filesystem at 1731899690301Initializing all the Stores at 1731899690303 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899690303Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899690304 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899690304Cleaning up temporary data from old regions at 1731899690325 (+21 ms)Running coprocessor post-open hooks at 1731899690333 (+8 ms)Region opened successfully at 1731899690334 (+1 ms) 2024-11-18T03:14:50,337 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., pid=6, masterSystemTime=1731899690291 2024-11-18T03:14:50,340 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:50,341 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:50,342 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=29141263847cbdef9bad2179de1271a3, regionState=OPEN, openSeqNum=2, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:50,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:14:50,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T03:14:50,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 in 212 msec 2024-11-18T03:14:50,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T03:14:50,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, ASSIGN in 384 msec 2024-11-18T03:14:50,360 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:14:50,360 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899690360"}]},"ts":"1731899690360"} 2024-11-18T03:14:50,364 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T03:14:50,366 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:14:50,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2670 sec 2024-11-18T03:14:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:14:51,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:14:51,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26071e9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a220135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,300 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322a9c3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,303 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,312 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:14:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fd862c3a04f9,45051,1731899683325 2024-11-18T03:14:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63481454 2024-11-18T03:14:51,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:14:51,323 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:14:51,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6c3e0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,382 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,402 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d7ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,404 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,406 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2685eeb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,427 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c1df973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,453 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ae7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,458 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d73963e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5224aa0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,466 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b688ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,470 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344fa400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730a2205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,481 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eac2906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,485 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a3e301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456ea33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788043de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,493 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421973e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3836c532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,500 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2572aff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70bfcb87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4ead28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:14:51,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:14:51,518 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:14:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:14:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:14:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676ef68d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:14:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:14:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:14:51,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:14:51,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5addfb03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:14:51,529 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,537 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,546 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,550 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,553 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:14:51,581 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-18T03:14:51,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,591 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T03:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,593 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:14:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:14:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:14:51,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,604 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,614 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:14:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:14:51,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:14:51,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:14:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:51,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:14:51,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T03:14:51,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T03:14:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:51,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:51,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:51,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/d68bd7437c37472cb77e3af39d635850 is 50, key is test_row_0/A:col10/1731899691664/Put/seqid=0 2024-11-18T03:14:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899751790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899751801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899751807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899751808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899751820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741837_1013 (size=12001) 2024-11-18T03:14:51,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/d68bd7437c37472cb77e3af39d635850 2024-11-18T03:14:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T03:14:51,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899751932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899751952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899751954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899751954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899751957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:51,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:51,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T03:14:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:51,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:51,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7bcf8b878e6b4936a8719f6599da2bb6 is 50, key is test_row_0/B:col10/1731899691664/Put/seqid=0 2024-11-18T03:14:52,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741838_1014 (size=12001) 2024-11-18T03:14:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899752143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T03:14:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:52,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899752169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899752170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899752174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899752174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T03:14:52,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T03:14:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7bcf8b878e6b4936a8719f6599da2bb6 2024-11-18T03:14:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899752452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T03:14:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899752482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899752483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899752492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/428ece1356ca475e9dfe0bff598f4888 is 50, key is test_row_0/C:col10/1731899691664/Put/seqid=0 2024-11-18T03:14:52,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899752492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:52,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741839_1015 (size=12001) 2024-11-18T03:14:52,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/428ece1356ca475e9dfe0bff598f4888 2024-11-18T03:14:52,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/d68bd7437c37472cb77e3af39d635850 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d68bd7437c37472cb77e3af39d635850 2024-11-18T03:14:52,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d68bd7437c37472cb77e3af39d635850, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T03:14:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7bcf8b878e6b4936a8719f6599da2bb6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7bcf8b878e6b4936a8719f6599da2bb6 2024-11-18T03:14:52,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7bcf8b878e6b4936a8719f6599da2bb6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T03:14:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/428ece1356ca475e9dfe0bff598f4888 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/428ece1356ca475e9dfe0bff598f4888 2024-11-18T03:14:52,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T03:14:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:52,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/428ece1356ca475e9dfe0bff598f4888, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T03:14:52,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 29141263847cbdef9bad2179de1271a3 in 1001ms, sequenceid=13, compaction requested=false 2024-11-18T03:14:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T03:14:52,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T03:14:52,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T03:14:52,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:52,798 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T03:14:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/e1e9b0a8a9b7478e8d3581836573be85 is 50, key is test_row_0/A:col10/1731899691804/Put/seqid=0 2024-11-18T03:14:52,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741840_1016 (size=12001) 2024-11-18T03:14:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:53,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899753026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899753027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899753039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899753032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899753044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899753162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899753162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899753172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899753172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899753173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,289 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/e1e9b0a8a9b7478e8d3581836573be85 2024-11-18T03:14:53,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/607314d38c694b1ba73f72c61232a116 is 50, key is test_row_0/B:col10/1731899691804/Put/seqid=0 2024-11-18T03:14:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741841_1017 (size=12001) 2024-11-18T03:14:53,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/607314d38c694b1ba73f72c61232a116 2024-11-18T03:14:53,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899753385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899753386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/6ba52b6551c444eeb8e48bd670691bc9 is 50, key is test_row_0/C:col10/1731899691804/Put/seqid=0 2024-11-18T03:14:53,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899753392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899753392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899753393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741842_1018 (size=12001) 2024-11-18T03:14:53,466 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T03:14:53,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899753702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899753711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899753712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899753713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899753721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T03:14:53,837 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/6ba52b6551c444eeb8e48bd670691bc9 2024-11-18T03:14:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/e1e9b0a8a9b7478e8d3581836573be85 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e1e9b0a8a9b7478e8d3581836573be85 2024-11-18T03:14:53,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e1e9b0a8a9b7478e8d3581836573be85, entries=150, sequenceid=33, filesize=11.7 K 2024-11-18T03:14:53,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/607314d38c694b1ba73f72c61232a116 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/607314d38c694b1ba73f72c61232a116 2024-11-18T03:14:53,902 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/607314d38c694b1ba73f72c61232a116, entries=150, sequenceid=33, filesize=11.7 K 2024-11-18T03:14:53,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/6ba52b6551c444eeb8e48bd670691bc9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6ba52b6551c444eeb8e48bd670691bc9 2024-11-18T03:14:53,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6ba52b6551c444eeb8e48bd670691bc9, entries=150, sequenceid=33, filesize=11.7 K 2024-11-18T03:14:53,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 29141263847cbdef9bad2179de1271a3 in 1144ms, sequenceid=33, compaction requested=false 2024-11-18T03:14:53,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:53,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:53,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-18T03:14:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-18T03:14:53,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T03:14:53,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3500 sec 2024-11-18T03:14:53,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3770 sec 2024-11-18T03:14:54,175 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:14:54,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:14:54,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:54,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:54,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:54,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:54,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:54,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:54,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91 is 50, key is test_row_0/A:col10/1731899693038/Put/seqid=0 2024-11-18T03:14:54,265 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T03:14:54,265 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T03:14:54,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T03:14:54,268 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T03:14:54,269 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T03:14:54,269 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T03:14:54,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T03:14:54,270 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T03:14:54,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741843_1019 (size=12001) 2024-11-18T03:14:54,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91 2024-11-18T03:14:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899754298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899754311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899694291, max timestamp=1731899694314Num uniques 859; 2024-11-18T03:14:54,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:14:54,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899694291, max timestamp=1731899694314Num uniques 859; 2024-11-18T03:14:54,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:14:54,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/96f91653d6f9479e90e9e1259c75ab98 is 50, key is test_row_0/B:col10/1731899693038/Put/seqid=0 2024-11-18T03:14:54,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899694291, max timestamp=1731899694314Num uniques 859; 2024-11-18T03:14:54,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T03:14:54,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741844_1020 (size=12001) 2024-11-18T03:14:54,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/96f91653d6f9479e90e9e1259c75ab98 2024-11-18T03:14:54,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/193e35daca4d40268df4e1c52805823f is 50, key is test_row_0/C:col10/1731899693038/Put/seqid=0 2024-11-18T03:14:54,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899754440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899754453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899754458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741845_1021 (size=12001) 2024-11-18T03:14:54,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/193e35daca4d40268df4e1c52805823f 2024-11-18T03:14:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899754460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899754460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91 2024-11-18T03:14:54,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91, entries=150, sequenceid=46, filesize=11.7 K 2024-11-18T03:14:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/96f91653d6f9479e90e9e1259c75ab98 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/96f91653d6f9479e90e9e1259c75ab98 2024-11-18T03:14:54,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/96f91653d6f9479e90e9e1259c75ab98, entries=150, sequenceid=46, filesize=11.7 K 2024-11-18T03:14:54,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/193e35daca4d40268df4e1c52805823f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/193e35daca4d40268df4e1c52805823f 2024-11-18T03:14:54,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/193e35daca4d40268df4e1c52805823f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-18T03:14:54,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=127.08 KB/130125 for 29141263847cbdef9bad2179de1271a3 in 335ms, sequenceid=46, compaction requested=true 2024-11-18T03:14:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:54,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:14:54,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:54,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:14:54,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:54,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:14:54,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:54,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-18T03:14:54,591 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:54,593 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:54,595 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:54,595 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:54,606 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:14:54,610 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:54,611 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7bcf8b878e6b4936a8719f6599da2bb6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/607314d38c694b1ba73f72c61232a116, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/96f91653d6f9479e90e9e1259c75ab98] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.2 K 2024-11-18T03:14:54,614 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bcf8b878e6b4936a8719f6599da2bb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731899691575 2024-11-18T03:14:54,615 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:14:54,615 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:54,615 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 607314d38c694b1ba73f72c61232a116, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731899691778 2024-11-18T03:14:54,615 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d68bd7437c37472cb77e3af39d635850, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e1e9b0a8a9b7478e8d3581836573be85, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.2 K 2024-11-18T03:14:54,616 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f91653d6f9479e90e9e1259c75ab98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731899693021 2024-11-18T03:14:54,616 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d68bd7437c37472cb77e3af39d635850, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731899691575 2024-11-18T03:14:54,618 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e9b0a8a9b7478e8d3581836573be85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731899691778 2024-11-18T03:14:54,619 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0e62f5f75f4ff5b2b52ad9a1d9bf91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731899693021 2024-11-18T03:14:54,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731899694452, max timestamp=1731899694579Num uniques 1018; 2024-11-18T03:14:54,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T03:14:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:54,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:14:54,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:54,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:14:54,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:54,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:14:54,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3f0135fb6c994311a136980c84384779 is 50, key is test_row_0/A:col10/1731899694579/Put/seqid=0 2024-11-18T03:14:54,664 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:54,666 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7a8c6e3db1f74356bcf09ce4870768c7 is 50, key is test_row_0/B:col10/1731899693038/Put/seqid=0 2024-11-18T03:14:54,673 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:54,674 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebf7a3480a774095ba21090adcd32bdb is 50, key is test_row_0/A:col10/1731899693038/Put/seqid=0 2024-11-18T03:14:54,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741846_1022 (size=12001) 2024-11-18T03:14:54,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3f0135fb6c994311a136980c84384779 2024-11-18T03:14:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741848_1024 (size=12104) 2024-11-18T03:14:54,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899754682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899754680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899754703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,731 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebf7a3480a774095ba21090adcd32bdb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebf7a3480a774095ba21090adcd32bdb 2024-11-18T03:14:54,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741847_1023 (size=12104) 2024-11-18T03:14:54,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899754718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899754719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/358068d6b611498b8e7553dbaf08e045 is 50, key is test_row_0/B:col10/1731899694579/Put/seqid=0 2024-11-18T03:14:54,755 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7a8c6e3db1f74356bcf09ce4870768c7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7a8c6e3db1f74356bcf09ce4870768c7 2024-11-18T03:14:54,760 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into ebf7a3480a774095ba21090adcd32bdb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:54,760 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:54,763 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899694568; duration=0sec 2024-11-18T03:14:54,764 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:54,764 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:14:54,764 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:54,772 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:54,772 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:14:54,772 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 7a8c6e3db1f74356bcf09ce4870768c7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:54,772 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:54,772 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:54,772 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899694589; duration=0sec 2024-11-18T03:14:54,772 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/428ece1356ca475e9dfe0bff598f4888, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6ba52b6551c444eeb8e48bd670691bc9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/193e35daca4d40268df4e1c52805823f] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.2 K 2024-11-18T03:14:54,773 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:54,773 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:14:54,774 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 428ece1356ca475e9dfe0bff598f4888, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731899691575 2024-11-18T03:14:54,779 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba52b6551c444eeb8e48bd670691bc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731899691778 2024-11-18T03:14:54,781 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 193e35daca4d40268df4e1c52805823f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731899693021 2024-11-18T03:14:54,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741849_1025 (size=12001) 2024-11-18T03:14:54,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/358068d6b611498b8e7553dbaf08e045 2024-11-18T03:14:54,819 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:54,820 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/626e20aabb0d415baf17a060eeda4365 is 50, key is test_row_0/C:col10/1731899693038/Put/seqid=0 2024-11-18T03:14:54,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/e94a709d22644250aab04692ca4510cf is 50, key is test_row_0/C:col10/1731899694579/Put/seqid=0 2024-11-18T03:14:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899754831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899754831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741850_1026 (size=12104) 2024-11-18T03:14:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899754832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,858 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/626e20aabb0d415baf17a060eeda4365 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/626e20aabb0d415baf17a060eeda4365 2024-11-18T03:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899754852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899754852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:54,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741851_1027 (size=12001) 2024-11-18T03:14:54,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/e94a709d22644250aab04692ca4510cf 2024-11-18T03:14:54,874 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 626e20aabb0d415baf17a060eeda4365(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:54,874 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:54,874 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899694589; duration=0sec 2024-11-18T03:14:54,875 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:54,875 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:14:54,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3f0135fb6c994311a136980c84384779 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3f0135fb6c994311a136980c84384779 2024-11-18T03:14:54,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3f0135fb6c994311a136980c84384779, entries=150, sequenceid=84, filesize=11.7 K 2024-11-18T03:14:54,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/358068d6b611498b8e7553dbaf08e045 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/358068d6b611498b8e7553dbaf08e045 2024-11-18T03:14:54,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/358068d6b611498b8e7553dbaf08e045, entries=150, sequenceid=84, filesize=11.7 K 2024-11-18T03:14:54,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/e94a709d22644250aab04692ca4510cf as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/e94a709d22644250aab04692ca4510cf 2024-11-18T03:14:54,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/e94a709d22644250aab04692ca4510cf, entries=150, sequenceid=84, filesize=11.7 K 2024-11-18T03:14:54,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.48 KB/104935, heapSize ~291.24 KB/298232, currentSize=80.51 KB/82440 for 29141263847cbdef9bad2179de1271a3 in 354ms, sequenceid=84, compaction requested=false 2024-11-18T03:14:54,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:55,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:55,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/472935e907a745e08c67d37479aacbba is 50, key is test_row_0/A:col10/1731899695054/Put/seqid=0 2024-11-18T03:14:55,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899755108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899755105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741852_1028 (size=12001) 2024-11-18T03:14:55,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899755125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899755125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899755126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899755232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899755232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899755242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899755244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899755244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899755441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899755453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899755453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899755456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899755456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/472935e907a745e08c67d37479aacbba 2024-11-18T03:14:55,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/ab00b3a0aa3a426c86a7013a80304f37 is 50, key is test_row_0/B:col10/1731899695054/Put/seqid=0 2024-11-18T03:14:55,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741853_1029 (size=12001) 2024-11-18T03:14:55,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/ab00b3a0aa3a426c86a7013a80304f37 2024-11-18T03:14:55,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/45063f8e87a04bc5b13197a1467cb32a is 50, key is test_row_0/C:col10/1731899695054/Put/seqid=0 2024-11-18T03:14:55,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741854_1030 (size=12001) 2024-11-18T03:14:55,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/45063f8e87a04bc5b13197a1467cb32a 2024-11-18T03:14:55,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/472935e907a745e08c67d37479aacbba as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/472935e907a745e08c67d37479aacbba 2024-11-18T03:14:55,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/472935e907a745e08c67d37479aacbba, entries=150, sequenceid=104, filesize=11.7 K 2024-11-18T03:14:55,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/ab00b3a0aa3a426c86a7013a80304f37 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ab00b3a0aa3a426c86a7013a80304f37 2024-11-18T03:14:55,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ab00b3a0aa3a426c86a7013a80304f37, entries=150, sequenceid=104, filesize=11.7 K 2024-11-18T03:14:55,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/45063f8e87a04bc5b13197a1467cb32a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/45063f8e87a04bc5b13197a1467cb32a 2024-11-18T03:14:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899755751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T03:14:55,761 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:14:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:14:55,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899755761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-18T03:14:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899755761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T03:14:55,765 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:14:55,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899755765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,768 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:14:55,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:14:55,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/45063f8e87a04bc5b13197a1467cb32a, entries=150, sequenceid=104, filesize=11.7 K 2024-11-18T03:14:55,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 29141263847cbdef9bad2179de1271a3 in 709ms, sequenceid=104, compaction requested=true 2024-11-18T03:14:55,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:55,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:14:55,771 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:55,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:55,771 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:55,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:14:55,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:55,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:14:55,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:55,775 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:55,775 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:14:55,775 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:55,776 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7a8c6e3db1f74356bcf09ce4870768c7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/358068d6b611498b8e7553dbaf08e045, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ab00b3a0aa3a426c86a7013a80304f37] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.3 K 2024-11-18T03:14:55,776 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:55,776 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:14:55,776 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:55,776 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebf7a3480a774095ba21090adcd32bdb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3f0135fb6c994311a136980c84384779, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/472935e907a745e08c67d37479aacbba] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.3 K 2024-11-18T03:14:55,781 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a8c6e3db1f74356bcf09ce4870768c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731899693021 2024-11-18T03:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:55,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:14:55,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:55,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:55,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:55,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:55,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:55,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:55,783 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf7a3480a774095ba21090adcd32bdb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731899693021 2024-11-18T03:14:55,786 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 358068d6b611498b8e7553dbaf08e045, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731899694452 2024-11-18T03:14:55,787 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f0135fb6c994311a136980c84384779, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731899694452 2024-11-18T03:14:55,788 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ab00b3a0aa3a426c86a7013a80304f37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731899694695 2024-11-18T03:14:55,788 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 472935e907a745e08c67d37479aacbba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731899694695 2024-11-18T03:14:55,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/eef21d36af404c02bc666f74f99719d6 is 50, key is test_row_0/A:col10/1731899695780/Put/seqid=0 2024-11-18T03:14:55,823 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:55,824 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/7c7c9153f60e48348b8347abf2e948e0 is 50, key is test_row_0/A:col10/1731899695054/Put/seqid=0 2024-11-18T03:14:55,825 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:55,826 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/c9ab43e455d54b24ab07c8af751071d4 is 50, key is test_row_0/B:col10/1731899695054/Put/seqid=0 2024-11-18T03:14:55,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741855_1031 (size=12001) 2024-11-18T03:14:55,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/eef21d36af404c02bc666f74f99719d6 2024-11-18T03:14:55,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741856_1032 (size=12207) 2024-11-18T03:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T03:14:55,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/68098dd21d554638841288becb72daa2 is 50, key is test_row_0/B:col10/1731899695780/Put/seqid=0 2024-11-18T03:14:55,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899755878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:55,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741857_1033 (size=12207) 2024-11-18T03:14:55,920 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/7c7c9153f60e48348b8347abf2e948e0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7c7c9153f60e48348b8347abf2e948e0 2024-11-18T03:14:55,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:55,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:55,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741858_1034 (size=12001) 2024-11-18T03:14:55,940 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 7c7c9153f60e48348b8347abf2e948e0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:55,941 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:55,941 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899695770; duration=0sec 2024-11-18T03:14:55,942 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:55,942 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:14:55,942 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:55,944 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:55,944 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:14:55,945 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:55,945 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/626e20aabb0d415baf17a060eeda4365, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/e94a709d22644250aab04692ca4510cf, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/45063f8e87a04bc5b13197a1467cb32a] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.3 K 2024-11-18T03:14:55,947 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 626e20aabb0d415baf17a060eeda4365, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731899693021 2024-11-18T03:14:55,948 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e94a709d22644250aab04692ca4510cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731899694452 2024-11-18T03:14:55,949 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45063f8e87a04bc5b13197a1467cb32a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731899694695 2024-11-18T03:14:55,972 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:55,977 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/192f764bebc646f79dfeb5ec51b5003f is 50, key is test_row_0/C:col10/1731899695054/Put/seqid=0 2024-11-18T03:14:55,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899755991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741859_1035 (size=12207) 2024-11-18T03:14:56,040 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/192f764bebc646f79dfeb5ec51b5003f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/192f764bebc646f79dfeb5ec51b5003f 2024-11-18T03:14:56,061 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 192f764bebc646f79dfeb5ec51b5003f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:56,061 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:56,061 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899695771; duration=0sec 2024-11-18T03:14:56,062 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:56,062 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:14:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T03:14:56,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899756201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:56,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899756272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899756272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899756273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899756284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,298 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/c9ab43e455d54b24ab07c8af751071d4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c9ab43e455d54b24ab07c8af751071d4 2024-11-18T03:14:56,317 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into c9ab43e455d54b24ab07c8af751071d4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:56,317 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:56,317 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899695771; duration=0sec 2024-11-18T03:14:56,318 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:56,318 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:14:56,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/68098dd21d554638841288becb72daa2 2024-11-18T03:14:56,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/7e468e3d1b9945e191b1a581f383a023 is 50, key is test_row_0/C:col10/1731899695780/Put/seqid=0 2024-11-18T03:14:56,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741860_1036 (size=12001) 2024-11-18T03:14:56,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/7e468e3d1b9945e191b1a581f383a023 2024-11-18T03:14:56,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/eef21d36af404c02bc666f74f99719d6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/eef21d36af404c02bc666f74f99719d6 2024-11-18T03:14:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T03:14:56,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/eef21d36af404c02bc666f74f99719d6, entries=150, sequenceid=122, filesize=11.7 K 2024-11-18T03:14:56,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/68098dd21d554638841288becb72daa2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/68098dd21d554638841288becb72daa2 2024-11-18T03:14:56,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/68098dd21d554638841288becb72daa2, entries=150, sequenceid=122, filesize=11.7 K 2024-11-18T03:14:56,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/7e468e3d1b9945e191b1a581f383a023 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7e468e3d1b9945e191b1a581f383a023 2024-11-18T03:14:56,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7e468e3d1b9945e191b1a581f383a023, entries=150, sequenceid=122, filesize=11.7 K 2024-11-18T03:14:56,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 29141263847cbdef9bad2179de1271a3 in 665ms, sequenceid=122, compaction requested=false 2024-11-18T03:14:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:56,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:14:56,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:56,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:56,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:56,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a4c1ac5e9e004dc8bc6ec54a5605370a is 50, key is test_row_0/A:col10/1731899696526/Put/seqid=0 2024-11-18T03:14:56,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741861_1037 (size=12151) 2024-11-18T03:14:56,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a4c1ac5e9e004dc8bc6ec54a5605370a 2024-11-18T03:14:56,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/4c9b1bcbb2c241bc9132dc4904ddaf86 is 50, key is test_row_0/B:col10/1731899696526/Put/seqid=0 2024-11-18T03:14:56,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741862_1038 (size=12151) 2024-11-18T03:14:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899756669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899756781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:56,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:56,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:56,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:56,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T03:14:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899756992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:57,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:57,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:57,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/4c9b1bcbb2c241bc9132dc4904ddaf86 2024-11-18T03:14:57,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/862c2296cd9c4496b875664813aabd1c is 50, key is test_row_0/C:col10/1731899696526/Put/seqid=0 2024-11-18T03:14:57,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741863_1039 (size=12151) 2024-11-18T03:14:57,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/862c2296cd9c4496b875664813aabd1c 2024-11-18T03:14:57,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a4c1ac5e9e004dc8bc6ec54a5605370a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a4c1ac5e9e004dc8bc6ec54a5605370a 2024-11-18T03:14:57,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a4c1ac5e9e004dc8bc6ec54a5605370a, entries=150, sequenceid=140, filesize=11.9 K 2024-11-18T03:14:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/4c9b1bcbb2c241bc9132dc4904ddaf86 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/4c9b1bcbb2c241bc9132dc4904ddaf86 2024-11-18T03:14:57,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/4c9b1bcbb2c241bc9132dc4904ddaf86, entries=150, sequenceid=140, filesize=11.9 K 2024-11-18T03:14:57,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/862c2296cd9c4496b875664813aabd1c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/862c2296cd9c4496b875664813aabd1c 2024-11-18T03:14:57,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/862c2296cd9c4496b875664813aabd1c, entries=150, sequenceid=140, filesize=11.9 K 2024-11-18T03:14:57,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 29141263847cbdef9bad2179de1271a3 in 604ms, sequenceid=140, compaction requested=true 2024-11-18T03:14:57,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:57,135 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:57,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:14:57,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:57,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:14:57,136 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:57,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:57,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:14:57,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:57,138 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:57,138 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:57,138 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:14:57,138 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:14:57,138 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:57,138 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:57,138 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7c7c9153f60e48348b8347abf2e948e0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/eef21d36af404c02bc666f74f99719d6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a4c1ac5e9e004dc8bc6ec54a5605370a] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.5 K 2024-11-18T03:14:57,138 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c9ab43e455d54b24ab07c8af751071d4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/68098dd21d554638841288becb72daa2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/4c9b1bcbb2c241bc9132dc4904ddaf86] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.5 K 2024-11-18T03:14:57,139 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ab43e455d54b24ab07c8af751071d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731899694695 2024-11-18T03:14:57,139 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7c9153f60e48348b8347abf2e948e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731899694695 2024-11-18T03:14:57,140 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting eef21d36af404c02bc666f74f99719d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731899695077 2024-11-18T03:14:57,140 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 68098dd21d554638841288becb72daa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731899695077 2024-11-18T03:14:57,141 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9b1bcbb2c241bc9132dc4904ddaf86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731899695860 2024-11-18T03:14:57,141 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c1ac5e9e004dc8bc6ec54a5605370a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731899695860 2024-11-18T03:14:57,163 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:57,164 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/b855b8dfd542454c8a7ca6aab7e6185b is 50, key is test_row_0/B:col10/1731899696526/Put/seqid=0 2024-11-18T03:14:57,165 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#27 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:57,167 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/efc7376868d4493687e27a4fbecb6668 is 50, key is test_row_0/A:col10/1731899696526/Put/seqid=0 2024-11-18T03:14:57,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T03:14:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:57,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:14:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:57,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741865_1041 (size=12459) 2024-11-18T03:14:57,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741864_1040 (size=12459) 2024-11-18T03:14:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/80afcf6978374a5a9f6bb22649893059 is 50, key is test_row_0/A:col10/1731899696656/Put/seqid=0 2024-11-18T03:14:57,263 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/b855b8dfd542454c8a7ca6aab7e6185b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b855b8dfd542454c8a7ca6aab7e6185b 2024-11-18T03:14:57,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741866_1042 (size=12151) 2024-11-18T03:14:57,291 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into b855b8dfd542454c8a7ca6aab7e6185b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:57,292 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:57,292 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899697135; duration=0sec 2024-11-18T03:14:57,292 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:57,292 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:14:57,292 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:57,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/80afcf6978374a5a9f6bb22649893059 2024-11-18T03:14:57,298 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:57,298 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:14:57,299 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:57,299 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/192f764bebc646f79dfeb5ec51b5003f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7e468e3d1b9945e191b1a581f383a023, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/862c2296cd9c4496b875664813aabd1c] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.5 K 2024-11-18T03:14:57,300 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 192f764bebc646f79dfeb5ec51b5003f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731899694695 2024-11-18T03:14:57,304 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e468e3d1b9945e191b1a581f383a023, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731899695077 2024-11-18T03:14:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:57,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:57,305 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 862c2296cd9c4496b875664813aabd1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731899695860 2024-11-18T03:14:57,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/17963c82defe4f5ca946ff9ce673497b is 50, key is test_row_0/B:col10/1731899696656/Put/seqid=0 2024-11-18T03:14:57,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899757331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899757335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899757338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,346 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:57,347 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/91bdcf6b9cb349e9860a5c07caacc7e2 is 50, key is test_row_0/C:col10/1731899696526/Put/seqid=0 2024-11-18T03:14:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899757343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899757340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741867_1043 (size=12151) 2024-11-18T03:14:57,368 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/17963c82defe4f5ca946ff9ce673497b 2024-11-18T03:14:57,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741868_1044 (size=12459) 2024-11-18T03:14:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d907a8d7e4ab4a939b29637a3fb2585c is 50, key is test_row_0/C:col10/1731899696656/Put/seqid=0 2024-11-18T03:14:57,406 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/91bdcf6b9cb349e9860a5c07caacc7e2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/91bdcf6b9cb349e9860a5c07caacc7e2 2024-11-18T03:14:57,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741869_1045 (size=12151) 2024-11-18T03:14:57,418 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d907a8d7e4ab4a939b29637a3fb2585c 2024-11-18T03:14:57,420 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 91bdcf6b9cb349e9860a5c07caacc7e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:57,420 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:57,420 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899697136; duration=0sec 2024-11-18T03:14:57,420 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:57,420 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:14:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/80afcf6978374a5a9f6bb22649893059 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/80afcf6978374a5a9f6bb22649893059 2024-11-18T03:14:57,443 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/80afcf6978374a5a9f6bb22649893059, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T03:14:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/17963c82defe4f5ca946ff9ce673497b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/17963c82defe4f5ca946ff9ce673497b 2024-11-18T03:14:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899757451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899757453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/17963c82defe4f5ca946ff9ce673497b, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T03:14:57,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d907a8d7e4ab4a939b29637a3fb2585c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d907a8d7e4ab4a939b29637a3fb2585c 2024-11-18T03:14:57,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899757473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899757473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899757473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,484 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d907a8d7e4ab4a939b29637a3fb2585c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T03:14:57,486 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 29141263847cbdef9bad2179de1271a3 in 305ms, sequenceid=157, compaction requested=false 2024-11-18T03:14:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:57,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-18T03:14:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-18T03:14:57,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-18T03:14:57,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7260 sec 2024-11-18T03:14:57,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7370 sec 2024-11-18T03:14:57,634 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/efc7376868d4493687e27a4fbecb6668 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/efc7376868d4493687e27a4fbecb6668 2024-11-18T03:14:57,657 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into efc7376868d4493687e27a4fbecb6668(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:57,657 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:57,658 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899697135; duration=0sec 2024-11-18T03:14:57,658 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:57,658 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:57,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T03:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:57,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:57,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/e466dadf26024f35a0a904d9467dd018 is 50, key is test_row_0/A:col10/1731899697662/Put/seqid=0 2024-11-18T03:14:57,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899757713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899757714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899757717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741870_1046 (size=14541) 2024-11-18T03:14:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899757721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899757721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899757831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899757831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899757832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899757841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899757842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T03:14:57,912 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:14:57,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:14:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-18T03:14:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T03:14:57,916 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:14:57,918 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:14:57,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:14:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T03:14:58,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899758041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899758051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899758051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899758062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899758062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T03:14:58,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:58,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/e466dadf26024f35a0a904d9467dd018 2024-11-18T03:14:58,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/3655cef67e834dcb8cfdc5822b506cb6 is 50, key is test_row_0/B:col10/1731899697662/Put/seqid=0 2024-11-18T03:14:58,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741871_1047 (size=12151) 2024-11-18T03:14:58,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/3655cef67e834dcb8cfdc5822b506cb6 2024-11-18T03:14:58,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d310cdad4f664f9c81738610e4f020ae is 50, key is test_row_0/C:col10/1731899697662/Put/seqid=0 2024-11-18T03:14:58,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T03:14:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T03:14:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741872_1048 (size=12151) 2024-11-18T03:14:58,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d310cdad4f664f9c81738610e4f020ae 2024-11-18T03:14:58,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/e466dadf26024f35a0a904d9467dd018 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e466dadf26024f35a0a904d9467dd018 2024-11-18T03:14:58,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e466dadf26024f35a0a904d9467dd018, entries=200, sequenceid=176, filesize=14.2 K 2024-11-18T03:14:58,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/3655cef67e834dcb8cfdc5822b506cb6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3655cef67e834dcb8cfdc5822b506cb6 2024-11-18T03:14:58,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3655cef67e834dcb8cfdc5822b506cb6, entries=150, sequenceid=176, filesize=11.9 K 2024-11-18T03:14:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d310cdad4f664f9c81738610e4f020ae as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d310cdad4f664f9c81738610e4f020ae 2024-11-18T03:14:58,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d310cdad4f664f9c81738610e4f020ae, entries=150, sequenceid=176, filesize=11.9 K 2024-11-18T03:14:58,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 29141263847cbdef9bad2179de1271a3 in 673ms, sequenceid=176, compaction requested=true 2024-11-18T03:14:58,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:58,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:14:58,342 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:58,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:58,343 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:58,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:14:58,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:58,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:14:58,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:58,344 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:58,344 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:14:58,344 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,345 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/efc7376868d4493687e27a4fbecb6668, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/80afcf6978374a5a9f6bb22649893059, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e466dadf26024f35a0a904d9467dd018] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=38.2 K 2024-11-18T03:14:58,346 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc7376868d4493687e27a4fbecb6668, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731899695860 2024-11-18T03:14:58,347 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:58,347 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:14:58,347 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,347 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b855b8dfd542454c8a7ca6aab7e6185b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/17963c82defe4f5ca946ff9ce673497b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3655cef67e834dcb8cfdc5822b506cb6] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.9 K 2024-11-18T03:14:58,348 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80afcf6978374a5a9f6bb22649893059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731899696606 2024-11-18T03:14:58,349 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b855b8dfd542454c8a7ca6aab7e6185b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731899695860 2024-11-18T03:14:58,349 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e466dadf26024f35a0a904d9467dd018, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731899697312 2024-11-18T03:14:58,351 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 17963c82defe4f5ca946ff9ce673497b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731899696606 2024-11-18T03:14:58,352 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3655cef67e834dcb8cfdc5822b506cb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731899697335 2024-11-18T03:14:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:58,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:14:58,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:58,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:58,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:58,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:58,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:58,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:58,371 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:58,375 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2339e197b61d456ea4cd49696f5c4c7f is 50, key is test_row_0/A:col10/1731899697662/Put/seqid=0 2024-11-18T03:14:58,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T03:14:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,386 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,387 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/57c505a808894099b9de72dfdd8a4f96 is 50, key is test_row_0/B:col10/1731899697662/Put/seqid=0 2024-11-18T03:14:58,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2ffd7ae071ca46f9bcd2489d34a79923 is 50, key is test_row_0/A:col10/1731899697709/Put/seqid=0 2024-11-18T03:14:58,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899758397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899758398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899758393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899758408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741873_1049 (size=12561) 2024-11-18T03:14:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899758409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,424 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2339e197b61d456ea4cd49696f5c4c7f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2339e197b61d456ea4cd49696f5c4c7f 2024-11-18T03:14:58,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741875_1051 (size=14541) 2024-11-18T03:14:58,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2ffd7ae071ca46f9bcd2489d34a79923 2024-11-18T03:14:58,439 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 2339e197b61d456ea4cd49696f5c4c7f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:58,439 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:58,439 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899698342; duration=0sec 2024-11-18T03:14:58,440 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:14:58,440 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:14:58,440 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:14:58,442 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:14:58,442 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:14:58,442 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,443 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/91bdcf6b9cb349e9860a5c07caacc7e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d907a8d7e4ab4a939b29637a3fb2585c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d310cdad4f664f9c81738610e4f020ae] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=35.9 K 2024-11-18T03:14:58,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/eeb0d115222a4c28b1726ee367a5d7f2 is 50, key is test_row_0/B:col10/1731899697709/Put/seqid=0 2024-11-18T03:14:58,443 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91bdcf6b9cb349e9860a5c07caacc7e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731899695860 2024-11-18T03:14:58,445 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d907a8d7e4ab4a939b29637a3fb2585c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731899696606 2024-11-18T03:14:58,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741874_1050 (size=12561) 2024-11-18T03:14:58,449 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d310cdad4f664f9c81738610e4f020ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731899697335 2024-11-18T03:14:58,479 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:14:58,479 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/031340de9a424b4284bd45e76c622029 is 50, key is test_row_0/C:col10/1731899697662/Put/seqid=0 2024-11-18T03:14:58,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741876_1052 (size=12151) 2024-11-18T03:14:58,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/eeb0d115222a4c28b1726ee367a5d7f2 2024-11-18T03:14:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899758511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899758512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899758512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899758523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899758523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/ff222f0837cc4fb48d0734537ca42cd4 is 50, key is test_row_0/C:col10/1731899697709/Put/seqid=0 2024-11-18T03:14:58,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T03:14:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T03:14:58,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:14:58,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741877_1053 (size=12561) 2024-11-18T03:14:58,567 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/031340de9a424b4284bd45e76c622029 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/031340de9a424b4284bd45e76c622029 2024-11-18T03:14:58,577 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 031340de9a424b4284bd45e76c622029(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:58,578 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:58,578 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899698343; duration=0sec 2024-11-18T03:14:58,578 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:58,578 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:14:58,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741878_1054 (size=12151) 2024-11-18T03:14:58,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/ff222f0837cc4fb48d0734537ca42cd4 2024-11-18T03:14:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2ffd7ae071ca46f9bcd2489d34a79923 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2ffd7ae071ca46f9bcd2489d34a79923 2024-11-18T03:14:58,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2ffd7ae071ca46f9bcd2489d34a79923, entries=200, sequenceid=194, filesize=14.2 K 2024-11-18T03:14:58,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/eeb0d115222a4c28b1726ee367a5d7f2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/eeb0d115222a4c28b1726ee367a5d7f2 2024-11-18T03:14:58,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/eeb0d115222a4c28b1726ee367a5d7f2, entries=150, sequenceid=194, filesize=11.9 K 2024-11-18T03:14:58,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/ff222f0837cc4fb48d0734537ca42cd4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/ff222f0837cc4fb48d0734537ca42cd4 2024-11-18T03:14:58,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/ff222f0837cc4fb48d0734537ca42cd4, entries=150, sequenceid=194, filesize=11.9 K 2024-11-18T03:14:58,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 29141263847cbdef9bad2179de1271a3 in 292ms, sequenceid=194, compaction requested=false 2024-11-18T03:14:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:58,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T03:14:58,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:58,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T03:14:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:58,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:58,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:58,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:58,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:58,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d is 50, key is test_row_0/A:col10/1731899698404/Put/seqid=0 2024-11-18T03:14:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:14:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:58,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741879_1055 (size=12151) 2024-11-18T03:14:58,776 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d 2024-11-18T03:14:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899758775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899758775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899758785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899758789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899758790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/cb97d505869341f2ae6c08930afaf594 is 50, key is test_row_0/B:col10/1731899698404/Put/seqid=0 2024-11-18T03:14:58,856 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/57c505a808894099b9de72dfdd8a4f96 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/57c505a808894099b9de72dfdd8a4f96 2024-11-18T03:14:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741880_1056 (size=12151) 2024-11-18T03:14:58,865 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 57c505a808894099b9de72dfdd8a4f96(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:14:58,865 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:58,866 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899698343; duration=0sec 2024-11-18T03:14:58,866 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:14:58,866 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:14:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899758902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899758903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899758902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899758911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899758911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T03:14:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899759112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899759121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899759123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899759123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899759141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/cb97d505869341f2ae6c08930afaf594 2024-11-18T03:14:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9e5c87aa42654a4482094483c40fd72d is 50, key is test_row_0/C:col10/1731899698404/Put/seqid=0 2024-11-18T03:14:59,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741881_1057 (size=12151) 2024-11-18T03:14:59,323 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9e5c87aa42654a4482094483c40fd72d 2024-11-18T03:14:59,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d 2024-11-18T03:14:59,353 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d, entries=150, sequenceid=210, filesize=11.9 K 2024-11-18T03:14:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/cb97d505869341f2ae6c08930afaf594 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/cb97d505869341f2ae6c08930afaf594 2024-11-18T03:14:59,376 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/cb97d505869341f2ae6c08930afaf594, entries=150, sequenceid=210, filesize=11.9 K 2024-11-18T03:14:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9e5c87aa42654a4482094483c40fd72d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9e5c87aa42654a4482094483c40fd72d 2024-11-18T03:14:59,391 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9e5c87aa42654a4482094483c40fd72d, entries=150, sequenceid=210, filesize=11.9 K 2024-11-18T03:14:59,393 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 29141263847cbdef9bad2179de1271a3 in 693ms, sequenceid=210, compaction requested=true 2024-11-18T03:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-18T03:14:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-18T03:14:59,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T03:14:59,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4780 sec 2024-11-18T03:14:59,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4910 sec 2024-11-18T03:14:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:14:59,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T03:14:59,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:14:59,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:59,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:14:59,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:59,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:14:59,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:14:59,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/73e29039d9474894b341e283cee7ccb8 is 50, key is test_row_0/A:col10/1731899698787/Put/seqid=0 2024-11-18T03:14:59,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899759481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899759481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899759484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741882_1058 (size=14541) 2024-11-18T03:14:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899759493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899759498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899759613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899759616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899759616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899759616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899759616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899759831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899759832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899759832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899759832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899759836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:14:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:14:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:14:59,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/73e29039d9474894b341e283cee7ccb8 2024-11-18T03:14:59,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/f6c1d9be00e44647bbd4a82aa1e795bb is 50, key is test_row_0/B:col10/1731899698787/Put/seqid=0 2024-11-18T03:14:59,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741883_1059 (size=12151) 2024-11-18T03:15:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T03:15:00,062 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:00,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-18T03:15:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T03:15:00,071 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:00,073 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:00,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:00,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899760144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899760151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899760144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899760151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899760153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T03:15:00,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T03:15:00,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/f6c1d9be00e44647bbd4a82aa1e795bb 2024-11-18T03:15:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T03:15:00,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/866e63474b284ab6886eb1bc49ca9815 is 50, key is test_row_0/C:col10/1731899698787/Put/seqid=0 2024-11-18T03:15:00,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T03:15:00,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:00,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741884_1060 (size=12151) 2024-11-18T03:15:00,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899760671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899760672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899760672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899760672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899760673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T03:15:00,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T03:15:00,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/866e63474b284ab6886eb1bc49ca9815 2024-11-18T03:15:00,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/73e29039d9474894b341e283cee7ccb8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/73e29039d9474894b341e283cee7ccb8 2024-11-18T03:15:00,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/73e29039d9474894b341e283cee7ccb8, entries=200, sequenceid=230, filesize=14.2 K 2024-11-18T03:15:00,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/f6c1d9be00e44647bbd4a82aa1e795bb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f6c1d9be00e44647bbd4a82aa1e795bb 2024-11-18T03:15:00,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T03:15:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:00,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f6c1d9be00e44647bbd4a82aa1e795bb, entries=150, sequenceid=230, filesize=11.9 K 2024-11-18T03:15:00,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/866e63474b284ab6886eb1bc49ca9815 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/866e63474b284ab6886eb1bc49ca9815 2024-11-18T03:15:00,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/866e63474b284ab6886eb1bc49ca9815, entries=150, sequenceid=230, filesize=11.9 K 2024-11-18T03:15:00,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 29141263847cbdef9bad2179de1271a3 in 1482ms, sequenceid=230, compaction requested=true 2024-11-18T03:15:00,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:00,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:00,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:15:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:00,912 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:15:00,916 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:00,920 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:00,920 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:00,920 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:00,920 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:00,920 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,920 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/57c505a808894099b9de72dfdd8a4f96, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/eeb0d115222a4c28b1726ee367a5d7f2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/cb97d505869341f2ae6c08930afaf594, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f6c1d9be00e44647bbd4a82aa1e795bb] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=47.9 K 2024-11-18T03:15:00,921 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:00,921 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2339e197b61d456ea4cd49696f5c4c7f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2ffd7ae071ca46f9bcd2489d34a79923, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/73e29039d9474894b341e283cee7ccb8] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=52.5 K 2024-11-18T03:15:00,921 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c505a808894099b9de72dfdd8a4f96, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731899697335 2024-11-18T03:15:00,921 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2339e197b61d456ea4cd49696f5c4c7f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731899697335 2024-11-18T03:15:00,922 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb0d115222a4c28b1726ee367a5d7f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731899697709 2024-11-18T03:15:00,922 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ffd7ae071ca46f9bcd2489d34a79923, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731899697709 2024-11-18T03:15:00,922 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7e2d06bfc4e4cf0853eab9b41b8ff8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731899698390 2024-11-18T03:15:00,922 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting cb97d505869341f2ae6c08930afaf594, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731899698390 2024-11-18T03:15:00,923 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e29039d9474894b341e283cee7ccb8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731899698780 2024-11-18T03:15:00,923 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f6c1d9be00e44647bbd4a82aa1e795bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731899698780 2024-11-18T03:15:00,956 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:00,957 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7410d835eac14187be3a186f0943a920 is 50, key is test_row_0/B:col10/1731899698787/Put/seqid=0 2024-11-18T03:15:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741885_1061 (size=12697) 2024-11-18T03:15:00,975 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#49 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:00,976 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a16cc9eceae2437e965a56aeecc2d961 is 50, key is test_row_0/A:col10/1731899698787/Put/seqid=0 2024-11-18T03:15:01,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741886_1062 (size=12697) 2024-11-18T03:15:01,035 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a16cc9eceae2437e965a56aeecc2d961 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a16cc9eceae2437e965a56aeecc2d961 2024-11-18T03:15:01,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T03:15:01,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:01,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:15:01,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:01,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:01,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:01,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:01,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:01,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:01,053 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into a16cc9eceae2437e965a56aeecc2d961(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:01,053 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:01,053 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=12, startTime=1731899700911; duration=0sec 2024-11-18T03:15:01,054 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:01,054 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:01,054 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:01,056 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:01,056 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:01,056 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:01,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/6067a57fa2b54cb8bd44cdbc13b77f39 is 50, key is test_row_0/A:col10/1731899699489/Put/seqid=0 2024-11-18T03:15:01,062 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/031340de9a424b4284bd45e76c622029, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/ff222f0837cc4fb48d0734537ca42cd4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9e5c87aa42654a4482094483c40fd72d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/866e63474b284ab6886eb1bc49ca9815] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=47.9 K 2024-11-18T03:15:01,063 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 031340de9a424b4284bd45e76c622029, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731899697335 2024-11-18T03:15:01,069 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff222f0837cc4fb48d0734537ca42cd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731899697709 2024-11-18T03:15:01,069 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5c87aa42654a4482094483c40fd72d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731899698390 2024-11-18T03:15:01,070 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 866e63474b284ab6886eb1bc49ca9815, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731899698780 2024-11-18T03:15:01,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741887_1063 (size=12151) 2024-11-18T03:15:01,126 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/6067a57fa2b54cb8bd44cdbc13b77f39 2024-11-18T03:15:01,128 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#51 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:01,128 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/4ff0f2428f304ba4a17946618743576b is 50, key is test_row_0/C:col10/1731899698787/Put/seqid=0 2024-11-18T03:15:01,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/dab91974521449d9b09aa7001f8887cd is 50, key is test_row_0/B:col10/1731899699489/Put/seqid=0 2024-11-18T03:15:01,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741888_1064 (size=12697) 2024-11-18T03:15:01,170 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/4ff0f2428f304ba4a17946618743576b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/4ff0f2428f304ba4a17946618743576b 2024-11-18T03:15:01,192 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 4ff0f2428f304ba4a17946618743576b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:01,192 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:01,192 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=12, startTime=1731899700912; duration=0sec 2024-11-18T03:15:01,193 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:01,193 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:01,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741889_1065 (size=12151) 2024-11-18T03:15:01,210 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/dab91974521449d9b09aa7001f8887cd 2024-11-18T03:15:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T03:15:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b36ca46d4fe841cd8023fff9110be6a3 is 50, key is test_row_0/C:col10/1731899699489/Put/seqid=0 2024-11-18T03:15:01,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741890_1066 (size=12151) 2024-11-18T03:15:01,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b36ca46d4fe841cd8023fff9110be6a3 2024-11-18T03:15:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/6067a57fa2b54cb8bd44cdbc13b77f39 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6067a57fa2b54cb8bd44cdbc13b77f39 2024-11-18T03:15:01,366 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6067a57fa2b54cb8bd44cdbc13b77f39, entries=150, sequenceid=243, filesize=11.9 K 2024-11-18T03:15:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/dab91974521449d9b09aa7001f8887cd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/dab91974521449d9b09aa7001f8887cd 2024-11-18T03:15:01,396 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7410d835eac14187be3a186f0943a920 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7410d835eac14187be3a186f0943a920 2024-11-18T03:15:01,413 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/dab91974521449d9b09aa7001f8887cd, entries=150, sequenceid=243, filesize=11.9 K 2024-11-18T03:15:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b36ca46d4fe841cd8023fff9110be6a3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b36ca46d4fe841cd8023fff9110be6a3 2024-11-18T03:15:01,421 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 7410d835eac14187be3a186f0943a920(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:01,421 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:01,421 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=12, startTime=1731899700911; duration=0sec 2024-11-18T03:15:01,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:01,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:01,453 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b36ca46d4fe841cd8023fff9110be6a3, entries=150, sequenceid=243, filesize=11.9 K 2024-11-18T03:15:01,459 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 29141263847cbdef9bad2179de1271a3 in 419ms, sequenceid=243, compaction requested=false 2024-11-18T03:15:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-18T03:15:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-18T03:15:01,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T03:15:01,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3890 sec 2024-11-18T03:15:01,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.4040 sec 2024-11-18T03:15:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:01,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:15:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:01,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/96866764676e4885a622c0b103933a26 is 50, key is test_row_0/A:col10/1731899701713/Put/seqid=0 2024-11-18T03:15:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741891_1067 (size=14591) 2024-11-18T03:15:01,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899701766, max timestamp=1731899701783Num uniques 859; 2024-11-18T03:15:01,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899701766, max timestamp=1731899701783Num uniques 859; 2024-11-18T03:15:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:01,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899701766, max timestamp=1731899701783Num uniques 859; 2024-11-18T03:15:01,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T03:15:01,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899761826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899761830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899761834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899761838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899761842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899761955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899761963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899761963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899761955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:01,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899761972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899762171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899762171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899762172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899762172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/96866764676e4885a622c0b103933a26 2024-11-18T03:15:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899762191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/ef2afe9dfcef488db159d4350b8b16a5 is 50, key is test_row_0/B:col10/1731899701713/Put/seqid=0 2024-11-18T03:15:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741892_1068 (size=12201) 2024-11-18T03:15:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T03:15:02,221 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:02,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-18T03:15:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T03:15:02,227 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:02,229 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:02,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T03:15:02,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T03:15:02,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:02,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899762482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899762483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899762483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899762492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899762504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:02,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T03:15:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T03:15:02,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/ef2afe9dfcef488db159d4350b8b16a5 2024-11-18T03:15:02,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/062787d7d84e4efba33cbd7c4bfa8fc8 is 50, key is test_row_0/C:col10/1731899701713/Put/seqid=0 2024-11-18T03:15:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741893_1069 (size=12201) 2024-11-18T03:15:02,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/062787d7d84e4efba33cbd7c4bfa8fc8 2024-11-18T03:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/96866764676e4885a622c0b103933a26 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/96866764676e4885a622c0b103933a26 2024-11-18T03:15:02,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/96866764676e4885a622c0b103933a26, entries=200, sequenceid=257, filesize=14.2 K 2024-11-18T03:15:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/ef2afe9dfcef488db159d4350b8b16a5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ef2afe9dfcef488db159d4350b8b16a5 2024-11-18T03:15:02,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T03:15:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:02,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ef2afe9dfcef488db159d4350b8b16a5, entries=150, sequenceid=257, filesize=11.9 K 2024-11-18T03:15:02,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/062787d7d84e4efba33cbd7c4bfa8fc8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/062787d7d84e4efba33cbd7c4bfa8fc8 2024-11-18T03:15:02,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/062787d7d84e4efba33cbd7c4bfa8fc8, entries=150, sequenceid=257, filesize=11.9 K 2024-11-18T03:15:02,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 29141263847cbdef9bad2179de1271a3 in 1035ms, sequenceid=257, compaction requested=true 2024-11-18T03:15:02,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:02,753 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:02,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:02,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:02,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:02,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:02,754 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:02,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:02,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:02,758 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:02,758 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:02,758 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,759 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a16cc9eceae2437e965a56aeecc2d961, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6067a57fa2b54cb8bd44cdbc13b77f39, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/96866764676e4885a622c0b103933a26] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=38.5 K 2024-11-18T03:15:02,759 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16cc9eceae2437e965a56aeecc2d961, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731899698780 2024-11-18T03:15:02,759 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:02,759 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:02,759 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,760 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7410d835eac14187be3a186f0943a920, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/dab91974521449d9b09aa7001f8887cd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ef2afe9dfcef488db159d4350b8b16a5] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=36.2 K 2024-11-18T03:15:02,760 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7410d835eac14187be3a186f0943a920, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731899698780 2024-11-18T03:15:02,760 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6067a57fa2b54cb8bd44cdbc13b77f39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731899699435 2024-11-18T03:15:02,761 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting dab91974521449d9b09aa7001f8887cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731899699435 2024-11-18T03:15:02,761 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96866764676e4885a622c0b103933a26, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731899701706 2024-11-18T03:15:02,761 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2afe9dfcef488db159d4350b8b16a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731899701706 2024-11-18T03:15:02,779 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:02,780 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7c6493b9c0a84bf5ba08928e0f0dd6e3 is 50, key is test_row_0/B:col10/1731899701713/Put/seqid=0 2024-11-18T03:15:02,783 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:02,783 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/23421a43b37c451094b5143de16ad359 is 50, key is test_row_0/A:col10/1731899701713/Put/seqid=0 2024-11-18T03:15:02,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741894_1070 (size=12849) 2024-11-18T03:15:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T03:15:02,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741895_1071 (size=12849) 2024-11-18T03:15:02,859 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/7c6493b9c0a84bf5ba08928e0f0dd6e3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7c6493b9c0a84bf5ba08928e0f0dd6e3 2024-11-18T03:15:02,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T03:15:02,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,862 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-18T03:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:02,868 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/23421a43b37c451094b5143de16ad359 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23421a43b37c451094b5143de16ad359 2024-11-18T03:15:02,876 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 7c6493b9c0a84bf5ba08928e0f0dd6e3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:02,876 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:02,876 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899702754; duration=0sec 2024-11-18T03:15:02,879 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:02,879 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:02,880 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:02,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/d271e25f84504a0484df64b8431126e5 is 50, key is test_row_0/A:col10/1731899701831/Put/seqid=0 2024-11-18T03:15:02,882 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:02,882 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:02,882 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:02,882 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/4ff0f2428f304ba4a17946618743576b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b36ca46d4fe841cd8023fff9110be6a3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/062787d7d84e4efba33cbd7c4bfa8fc8] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=36.2 K 2024-11-18T03:15:02,886 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ff0f2428f304ba4a17946618743576b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731899698780 2024-11-18T03:15:02,888 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b36ca46d4fe841cd8023fff9110be6a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731899699435 2024-11-18T03:15:02,889 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 062787d7d84e4efba33cbd7c4bfa8fc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731899701706 2024-11-18T03:15:02,892 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 23421a43b37c451094b5143de16ad359(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:02,892 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:02,893 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899702752; duration=0sec 2024-11-18T03:15:02,893 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:02,893 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:02,922 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:02,923 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/678a1b254fb64e6193e9e8df34718e09 is 50, key is test_row_0/C:col10/1731899701713/Put/seqid=0 2024-11-18T03:15:02,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741896_1072 (size=12301) 2024-11-18T03:15:02,945 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/d271e25f84504a0484df64b8431126e5 2024-11-18T03:15:02,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741897_1073 (size=12849) 2024-11-18T03:15:02,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/b74e3fe3598a4fae9aa0ee4600607e0d is 50, key is test_row_0/B:col10/1731899701831/Put/seqid=0 2024-11-18T03:15:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:02,997 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/678a1b254fb64e6193e9e8df34718e09 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/678a1b254fb64e6193e9e8df34718e09 2024-11-18T03:15:03,011 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 678a1b254fb64e6193e9e8df34718e09(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:03,011 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:03,011 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899702754; duration=0sec 2024-11-18T03:15:03,012 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:03,012 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:03,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899763012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899763011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899763016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899763018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899763019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741898_1074 (size=12301) 2024-11-18T03:15:03,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899763132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899763132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899763132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899763134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899763341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899763341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899763341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899763342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T03:15:03,435 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/b74e3fe3598a4fae9aa0ee4600607e0d 2024-11-18T03:15:03,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a8d3b6cb5c01476a8ec1ecec45617b32 is 50, key is test_row_0/C:col10/1731899701831/Put/seqid=0 2024-11-18T03:15:03,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741899_1075 (size=12301) 2024-11-18T03:15:03,468 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a8d3b6cb5c01476a8ec1ecec45617b32 2024-11-18T03:15:03,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/d271e25f84504a0484df64b8431126e5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d271e25f84504a0484df64b8431126e5 2024-11-18T03:15:03,506 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d271e25f84504a0484df64b8431126e5, entries=150, sequenceid=293, filesize=12.0 K 2024-11-18T03:15:03,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/b74e3fe3598a4fae9aa0ee4600607e0d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b74e3fe3598a4fae9aa0ee4600607e0d 2024-11-18T03:15:03,519 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b74e3fe3598a4fae9aa0ee4600607e0d, entries=150, sequenceid=293, filesize=12.0 K 2024-11-18T03:15:03,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a8d3b6cb5c01476a8ec1ecec45617b32 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a8d3b6cb5c01476a8ec1ecec45617b32 2024-11-18T03:15:03,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a8d3b6cb5c01476a8ec1ecec45617b32, entries=150, sequenceid=293, filesize=12.0 K 2024-11-18T03:15:03,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=60.38 KB/61830 for 29141263847cbdef9bad2179de1271a3 in 685ms, sequenceid=293, compaction requested=false 2024-11-18T03:15:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-18T03:15:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-18T03:15:03,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-18T03:15:03,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3190 sec 2024-11-18T03:15:03,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.3340 sec 2024-11-18T03:15:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:03,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:15:03,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:03,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:03,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:03,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:03,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:03,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:03,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/006b9c9f6ea143a0b8df947e1a3cd0af is 50, key is test_row_0/A:col10/1731899703655/Put/seqid=0 2024-11-18T03:15:03,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899763685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899763687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899763689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899763697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741900_1076 (size=12301) 2024-11-18T03:15:03,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/006b9c9f6ea143a0b8df947e1a3cd0af 2024-11-18T03:15:03,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/a5a72fce4e1c47d98d8405fd2e8c13ed is 50, key is test_row_0/B:col10/1731899703655/Put/seqid=0 2024-11-18T03:15:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741901_1077 (size=12301) 2024-11-18T03:15:03,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/a5a72fce4e1c47d98d8405fd2e8c13ed 2024-11-18T03:15:03,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/92906373dec84d84ab0f5170601ed0c7 is 50, key is test_row_0/C:col10/1731899703655/Put/seqid=0 2024-11-18T03:15:03,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899763802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741902_1078 (size=12301) 2024-11-18T03:15:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899763812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899763812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:03,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899763812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899764011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899764021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899764021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899764032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899764032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,043 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 29141263847cbdef9bad2179de1271a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/92906373dec84d84ab0f5170601ed0c7 2024-11-18T03:15:04,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/006b9c9f6ea143a0b8df947e1a3cd0af as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/006b9c9f6ea143a0b8df947e1a3cd0af 2024-11-18T03:15:04,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/006b9c9f6ea143a0b8df947e1a3cd0af, entries=150, sequenceid=310, filesize=12.0 K 2024-11-18T03:15:04,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/a5a72fce4e1c47d98d8405fd2e8c13ed as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a5a72fce4e1c47d98d8405fd2e8c13ed 2024-11-18T03:15:04,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a5a72fce4e1c47d98d8405fd2e8c13ed, entries=150, sequenceid=310, filesize=12.0 K 2024-11-18T03:15:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/92906373dec84d84ab0f5170601ed0c7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/92906373dec84d84ab0f5170601ed0c7 2024-11-18T03:15:04,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/92906373dec84d84ab0f5170601ed0c7, entries=150, sequenceid=310, filesize=12.0 K 2024-11-18T03:15:04,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 29141263847cbdef9bad2179de1271a3 in 590ms, sequenceid=310, compaction requested=true 2024-11-18T03:15:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:04,245 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:04,245 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:04,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:04,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:04,246 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:04,246 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:04,247 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:04,247 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:04,247 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,247 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,247 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23421a43b37c451094b5143de16ad359, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d271e25f84504a0484df64b8431126e5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/006b9c9f6ea143a0b8df947e1a3cd0af] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=36.6 K 2024-11-18T03:15:04,247 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7c6493b9c0a84bf5ba08928e0f0dd6e3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b74e3fe3598a4fae9aa0ee4600607e0d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a5a72fce4e1c47d98d8405fd2e8c13ed] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=36.6 K 2024-11-18T03:15:04,247 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6493b9c0a84bf5ba08928e0f0dd6e3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731899701706 2024-11-18T03:15:04,247 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23421a43b37c451094b5143de16ad359, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731899701706 2024-11-18T03:15:04,248 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b74e3fe3598a4fae9aa0ee4600607e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899701816 2024-11-18T03:15:04,248 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a5a72fce4e1c47d98d8405fd2e8c13ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899703009 2024-11-18T03:15:04,248 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d271e25f84504a0484df64b8431126e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899701816 2024-11-18T03:15:04,249 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 006b9c9f6ea143a0b8df947e1a3cd0af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899703009 2024-11-18T03:15:04,261 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:04,262 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/6157d9d6cc1f40c18945abf56ea9eb70 is 50, key is test_row_0/A:col10/1731899703655/Put/seqid=0 2024-11-18T03:15:04,274 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:04,275 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/02b9740feb144922bc592352e1362742 is 50, key is test_row_0/B:col10/1731899703655/Put/seqid=0 2024-11-18T03:15:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741904_1080 (size=13051) 2024-11-18T03:15:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741903_1079 (size=13051) 2024-11-18T03:15:04,323 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/6157d9d6cc1f40c18945abf56ea9eb70 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6157d9d6cc1f40c18945abf56ea9eb70 2024-11-18T03:15:04,324 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/02b9740feb144922bc592352e1362742 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/02b9740feb144922bc592352e1362742 2024-11-18T03:15:04,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T03:15:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:04,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a07e9561c47a4be4bbc378828e2b724c is 50, key is test_row_0/A:col10/1731899703678/Put/seqid=0 2024-11-18T03:15:04,345 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 6157d9d6cc1f40c18945abf56ea9eb70(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:04,345 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:04,345 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899704245; duration=0sec 2024-11-18T03:15:04,345 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:04,345 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:04,345 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:04,347 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 02b9740feb144922bc592352e1362742(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:04,347 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:04,348 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899704245; duration=0sec 2024-11-18T03:15:04,348 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:04,348 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:04,353 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:04,354 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:04,354 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,354 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/678a1b254fb64e6193e9e8df34718e09, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a8d3b6cb5c01476a8ec1ecec45617b32, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/92906373dec84d84ab0f5170601ed0c7] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=36.6 K 2024-11-18T03:15:04,355 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678a1b254fb64e6193e9e8df34718e09, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731899701706 2024-11-18T03:15:04,359 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d3b6cb5c01476a8ec1ecec45617b32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899701816 2024-11-18T03:15:04,361 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92906373dec84d84ab0f5170601ed0c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899703009 2024-11-18T03:15:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T03:15:04,371 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:04,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-18T03:15:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T03:15:04,376 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:04,377 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:04,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:04,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741905_1081 (size=14741) 2024-11-18T03:15:04,393 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:04,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a07e9561c47a4be4bbc378828e2b724c 2024-11-18T03:15:04,394 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/481e2742eb444d3d85a3a922340ab073 is 50, key is test_row_0/C:col10/1731899703655/Put/seqid=0 2024-11-18T03:15:04,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741906_1082 (size=13051) 2024-11-18T03:15:04,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/26c31026274c40d591adda31d15dd878 is 50, key is test_row_0/B:col10/1731899703678/Put/seqid=0 2024-11-18T03:15:04,429 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/481e2742eb444d3d85a3a922340ab073 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/481e2742eb444d3d85a3a922340ab073 2024-11-18T03:15:04,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899764421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899764422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899764425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899764423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741907_1083 (size=12301) 2024-11-18T03:15:04,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/26c31026274c40d591adda31d15dd878 2024-11-18T03:15:04,444 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 481e2742eb444d3d85a3a922340ab073(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:04,444 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:04,444 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899704245; duration=0sec 2024-11-18T03:15:04,445 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:04,445 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:04,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9038e49fc7a446978791ae69be12c75e is 50, key is test_row_0/C:col10/1731899703678/Put/seqid=0 2024-11-18T03:15:04,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741908_1084 (size=12301) 2024-11-18T03:15:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T03:15:04,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T03:15:04,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:04,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899764541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899764542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899764542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899764542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T03:15:04,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:04,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T03:15:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899764751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899764751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899764751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899764752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:04,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T03:15:04,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:04,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:04,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9038e49fc7a446978791ae69be12c75e 2024-11-18T03:15:04,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a07e9561c47a4be4bbc378828e2b724c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a07e9561c47a4be4bbc378828e2b724c 2024-11-18T03:15:04,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a07e9561c47a4be4bbc378828e2b724c, entries=200, sequenceid=329, filesize=14.4 K 2024-11-18T03:15:04,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/26c31026274c40d591adda31d15dd878 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/26c31026274c40d591adda31d15dd878 2024-11-18T03:15:04,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/26c31026274c40d591adda31d15dd878, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T03:15:04,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9038e49fc7a446978791ae69be12c75e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9038e49fc7a446978791ae69be12c75e 2024-11-18T03:15:04,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9038e49fc7a446978791ae69be12c75e, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T03:15:04,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 29141263847cbdef9bad2179de1271a3 in 574ms, sequenceid=329, compaction requested=false 2024-11-18T03:15:04,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:04,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T03:15:04,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:04,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:15:04,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:04,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:04,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:04,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:04,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:04,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:04,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3ce2f6a4befa4af8b9f9cf55f71487bb is 50, key is test_row_0/A:col10/1731899704351/Put/seqid=0 2024-11-18T03:15:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T03:15:05,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741909_1085 (size=12301) 2024-11-18T03:15:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:05,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:05,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899765105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899765112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899765114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899765113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899765223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899765231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899765231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899765232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3ce2f6a4befa4af8b9f9cf55f71487bb 2024-11-18T03:15:05,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/e40cafa0b8614024894458e372ec5d7e is 50, key is test_row_0/B:col10/1731899704351/Put/seqid=0 2024-11-18T03:15:05,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741910_1086 (size=12301) 2024-11-18T03:15:05,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899765431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899765440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899765441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899765451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T03:15:05,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899765760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899765761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899765762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899765764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:05,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/e40cafa0b8614024894458e372ec5d7e 2024-11-18T03:15:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/7c58d7792de44384a39f665cab98ded3 is 50, key is test_row_0/C:col10/1731899704351/Put/seqid=0 2024-11-18T03:15:05,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741911_1087 (size=12301) 2024-11-18T03:15:05,895 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/7c58d7792de44384a39f665cab98ded3 2024-11-18T03:15:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3ce2f6a4befa4af8b9f9cf55f71487bb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3ce2f6a4befa4af8b9f9cf55f71487bb 2024-11-18T03:15:05,915 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3ce2f6a4befa4af8b9f9cf55f71487bb, entries=150, sequenceid=345, filesize=12.0 K 2024-11-18T03:15:05,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/e40cafa0b8614024894458e372ec5d7e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e40cafa0b8614024894458e372ec5d7e 2024-11-18T03:15:05,925 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e40cafa0b8614024894458e372ec5d7e, entries=150, sequenceid=345, filesize=12.0 K 2024-11-18T03:15:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/7c58d7792de44384a39f665cab98ded3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7c58d7792de44384a39f665cab98ded3 2024-11-18T03:15:05,937 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7c58d7792de44384a39f665cab98ded3, entries=150, sequenceid=345, filesize=12.0 K 2024-11-18T03:15:05,938 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 29141263847cbdef9bad2179de1271a3 in 945ms, sequenceid=345, compaction requested=true 2024-11-18T03:15:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-18T03:15:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-18T03:15:05,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T03:15:05,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-11-18T03:15:05,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5740 sec 2024-11-18T03:15:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:06,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T03:15:06,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:06,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:06,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:06,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:06,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:06,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:06,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/cd3d2cbf463a4679b480153574149d8e is 50, key is test_row_0/A:col10/1731899706073/Put/seqid=0 2024-11-18T03:15:06,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741912_1088 (size=12301) 2024-11-18T03:15:06,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899766125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899766231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899766271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899766271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899766281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899766281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899766441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/cd3d2cbf463a4679b480153574149d8e 2024-11-18T03:15:06,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/e1cb665ea8ac4bf18b6f273253859c97 is 50, key is test_row_0/B:col10/1731899706073/Put/seqid=0 2024-11-18T03:15:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741913_1089 (size=12301) 2024-11-18T03:15:06,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/e1cb665ea8ac4bf18b6f273253859c97 2024-11-18T03:15:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T03:15:06,521 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:06,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-18T03:15:06,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:06,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:06,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T03:15:06,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a3b26e2c30254074a1b3451e8c0b4191 is 50, key is test_row_0/C:col10/1731899706073/Put/seqid=0 2024-11-18T03:15:06,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741914_1090 (size=12301) 2024-11-18T03:15:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T03:15:06,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T03:15:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:06,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899766760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:06,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T03:15:06,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:06,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:06,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:06,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T03:15:06,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a3b26e2c30254074a1b3451e8c0b4191 2024-11-18T03:15:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/cd3d2cbf463a4679b480153574149d8e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/cd3d2cbf463a4679b480153574149d8e 2024-11-18T03:15:06,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/cd3d2cbf463a4679b480153574149d8e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T03:15:06,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/e1cb665ea8ac4bf18b6f273253859c97 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e1cb665ea8ac4bf18b6f273253859c97 2024-11-18T03:15:06,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e1cb665ea8ac4bf18b6f273253859c97, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T03:15:06,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T03:15:06,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:06,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:06,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:06,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:06,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a3b26e2c30254074a1b3451e8c0b4191 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a3b26e2c30254074a1b3451e8c0b4191 2024-11-18T03:15:07,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a3b26e2c30254074a1b3451e8c0b4191, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T03:15:07,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 29141263847cbdef9bad2179de1271a3 in 961ms, sequenceid=365, compaction requested=true 2024-11-18T03:15:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:07,037 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:07,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:07,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:07,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:07,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:07,038 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:07,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:07,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:07,039 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:07,039 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:07,039 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:07,040 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6157d9d6cc1f40c18945abf56ea9eb70, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a07e9561c47a4be4bbc378828e2b724c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3ce2f6a4befa4af8b9f9cf55f71487bb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/cd3d2cbf463a4679b480153574149d8e] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=51.2 K 2024-11-18T03:15:07,040 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6157d9d6cc1f40c18945abf56ea9eb70, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899703009 2024-11-18T03:15:07,043 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a07e9561c47a4be4bbc378828e2b724c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731899703678 2024-11-18T03:15:07,043 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:07,043 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:07,043 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:07,043 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/02b9740feb144922bc592352e1362742, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/26c31026274c40d591adda31d15dd878, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e40cafa0b8614024894458e372ec5d7e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e1cb665ea8ac4bf18b6f273253859c97] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=48.8 K 2024-11-18T03:15:07,044 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce2f6a4befa4af8b9f9cf55f71487bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731899704351 2024-11-18T03:15:07,044 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02b9740feb144922bc592352e1362742, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899703009 2024-11-18T03:15:07,044 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26c31026274c40d591adda31d15dd878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731899703678 2024-11-18T03:15:07,045 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3d2cbf463a4679b480153574149d8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899705099 2024-11-18T03:15:07,045 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e40cafa0b8614024894458e372ec5d7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731899704351 2024-11-18T03:15:07,045 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1cb665ea8ac4bf18b6f273253859c97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899705099 2024-11-18T03:15:07,067 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:07,068 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/23ca083a5a8b4938a94d9c64afcfffcd is 50, key is test_row_0/A:col10/1731899706073/Put/seqid=0 2024-11-18T03:15:07,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741915_1091 (size=13187) 2024-11-18T03:15:07,078 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#78 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:07,082 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/35bfa4e2f11a4848a67e5ac41f8517fb is 50, key is test_row_0/B:col10/1731899706073/Put/seqid=0 2024-11-18T03:15:07,085 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/23ca083a5a8b4938a94d9c64afcfffcd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23ca083a5a8b4938a94d9c64afcfffcd 2024-11-18T03:15:07,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741916_1092 (size=13187) 2024-11-18T03:15:07,102 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 23ca083a5a8b4938a94d9c64afcfffcd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:07,102 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:07,103 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=12, startTime=1731899707036; duration=0sec 2024-11-18T03:15:07,104 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:07,104 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:07,104 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:07,109 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:07,109 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:07,110 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:07,110 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/481e2742eb444d3d85a3a922340ab073, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9038e49fc7a446978791ae69be12c75e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7c58d7792de44384a39f665cab98ded3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a3b26e2c30254074a1b3451e8c0b4191] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=48.8 K 2024-11-18T03:15:07,115 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 481e2742eb444d3d85a3a922340ab073, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899703009 2024-11-18T03:15:07,116 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9038e49fc7a446978791ae69be12c75e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731899703678 2024-11-18T03:15:07,116 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c58d7792de44384a39f665cab98ded3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731899704351 2024-11-18T03:15:07,117 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b26e2c30254074a1b3451e8c0b4191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899705099 2024-11-18T03:15:07,140 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:07,141 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/6f5a2d81b6ee428998a923273c5716dd is 50, key is test_row_0/C:col10/1731899706073/Put/seqid=0 2024-11-18T03:15:07,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T03:15:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:07,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T03:15:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:07,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:07,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741917_1093 (size=13187) 2024-11-18T03:15:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a6f224c59bc64d70aeec1676f9daae32 is 50, key is test_row_0/A:col10/1731899706108/Put/seqid=0 2024-11-18T03:15:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T03:15:07,156 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/6f5a2d81b6ee428998a923273c5716dd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6f5a2d81b6ee428998a923273c5716dd 2024-11-18T03:15:07,166 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 6f5a2d81b6ee428998a923273c5716dd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:07,166 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:07,166 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=12, startTime=1731899707038; duration=0sec 2024-11-18T03:15:07,166 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:07,166 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741918_1094 (size=12301) 2024-11-18T03:15:07,170 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a6f224c59bc64d70aeec1676f9daae32 2024-11-18T03:15:07,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/1d94b38041b04c1997ac4f5c313e1d61 is 50, key is test_row_0/B:col10/1731899706108/Put/seqid=0 2024-11-18T03:15:07,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741919_1095 (size=12301) 2024-11-18T03:15:07,193 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/1d94b38041b04c1997ac4f5c313e1d61 2024-11-18T03:15:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b312adf5c9154be4b478d333ce787b5e is 50, key is test_row_0/C:col10/1731899706108/Put/seqid=0 2024-11-18T03:15:07,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741920_1096 (size=12301) 2024-11-18T03:15:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899767363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899767366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899767368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899767380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899767384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899767491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899767491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899767492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899767492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899767505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,532 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/35bfa4e2f11a4848a67e5ac41f8517fb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/35bfa4e2f11a4848a67e5ac41f8517fb 2024-11-18T03:15:07,545 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 35bfa4e2f11a4848a67e5ac41f8517fb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:07,545 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:07,545 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=12, startTime=1731899707038; duration=0sec 2024-11-18T03:15:07,545 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:07,545 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:07,612 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b312adf5c9154be4b478d333ce787b5e 2024-11-18T03:15:07,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/a6f224c59bc64d70aeec1676f9daae32 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a6f224c59bc64d70aeec1676f9daae32 2024-11-18T03:15:07,650 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a6f224c59bc64d70aeec1676f9daae32, entries=150, sequenceid=378, filesize=12.0 K 2024-11-18T03:15:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/1d94b38041b04c1997ac4f5c313e1d61 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d94b38041b04c1997ac4f5c313e1d61 2024-11-18T03:15:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T03:15:07,662 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d94b38041b04c1997ac4f5c313e1d61, entries=150, sequenceid=378, filesize=12.0 K 2024-11-18T03:15:07,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b312adf5c9154be4b478d333ce787b5e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b312adf5c9154be4b478d333ce787b5e 2024-11-18T03:15:07,676 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b312adf5c9154be4b478d333ce787b5e, entries=150, sequenceid=378, filesize=12.0 K 2024-11-18T03:15:07,677 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 29141263847cbdef9bad2179de1271a3 in 532ms, sequenceid=378, compaction requested=false 2024-11-18T03:15:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-18T03:15:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-18T03:15:07,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T03:15:07,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1550 sec 2024-11-18T03:15:07,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1790 sec 2024-11-18T03:15:07,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899707352, max timestamp=1731899707703Num uniques 859; 2024-11-18T03:15:07,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:07,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899707352, max timestamp=1731899707703Num uniques 859; 2024-11-18T03:15:07,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:07,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-18T03:15:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:07,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:07,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:07,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:07,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:07,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/8329aeb1adcb48efb6bf0a97cabdfaf1 is 50, key is test_row_0/A:col10/1731899707703/Put/seqid=0 2024-11-18T03:15:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741921_1097 (size=17181) 2024-11-18T03:15:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899767769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899767771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899767777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899767780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899767785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899767891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899767911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899767911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899767922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899767924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899768106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899768121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899768121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899768141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899768141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/8329aeb1adcb48efb6bf0a97cabdfaf1 2024-11-18T03:15:08,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/47d82ebb78584ea8a3789e70b270f127 is 50, key is test_row_0/B:col10/1731899707703/Put/seqid=0 2024-11-18T03:15:08,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741922_1098 (size=12301) 2024-11-18T03:15:08,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899768420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899768431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899768431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899768451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899768451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/47d82ebb78584ea8a3789e70b270f127 2024-11-18T03:15:08,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a4a638b85051402fb69144e9c09e269d is 50, key is test_row_0/C:col10/1731899707703/Put/seqid=0 2024-11-18T03:15:08,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741923_1099 (size=12301) 2024-11-18T03:15:08,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a4a638b85051402fb69144e9c09e269d 2024-11-18T03:15:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/8329aeb1adcb48efb6bf0a97cabdfaf1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/8329aeb1adcb48efb6bf0a97cabdfaf1 2024-11-18T03:15:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T03:15:08,681 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:08,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-18T03:15:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T03:15:08,686 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:08,687 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:08,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:08,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/8329aeb1adcb48efb6bf0a97cabdfaf1, entries=250, sequenceid=402, filesize=16.8 K 2024-11-18T03:15:08,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/47d82ebb78584ea8a3789e70b270f127 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/47d82ebb78584ea8a3789e70b270f127 2024-11-18T03:15:08,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/47d82ebb78584ea8a3789e70b270f127, entries=150, sequenceid=402, filesize=12.0 K 2024-11-18T03:15:08,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/a4a638b85051402fb69144e9c09e269d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a4a638b85051402fb69144e9c09e269d 2024-11-18T03:15:08,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a4a638b85051402fb69144e9c09e269d, entries=150, sequenceid=402, filesize=12.0 K 2024-11-18T03:15:08,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 29141263847cbdef9bad2179de1271a3 in 1009ms, sequenceid=402, compaction requested=true 2024-11-18T03:15:08,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:08,717 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:08,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:08,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:08,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:08,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:08,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:08,718 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:08,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:08,719 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:08,719 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:08,719 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:08,719 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/35bfa4e2f11a4848a67e5ac41f8517fb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d94b38041b04c1997ac4f5c313e1d61, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/47d82ebb78584ea8a3789e70b270f127] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=36.9 K 2024-11-18T03:15:08,720 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:08,720 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:08,720 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:08,720 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23ca083a5a8b4938a94d9c64afcfffcd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a6f224c59bc64d70aeec1676f9daae32, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/8329aeb1adcb48efb6bf0a97cabdfaf1] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=41.7 K 2024-11-18T03:15:08,722 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ca083a5a8b4938a94d9c64afcfffcd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899705099 2024-11-18T03:15:08,722 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 35bfa4e2f11a4848a67e5ac41f8517fb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899705099 2024-11-18T03:15:08,722 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f224c59bc64d70aeec1676f9daae32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731899706108 2024-11-18T03:15:08,722 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d94b38041b04c1997ac4f5c313e1d61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731899706108 2024-11-18T03:15:08,723 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8329aeb1adcb48efb6bf0a97cabdfaf1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731899707352 2024-11-18T03:15:08,723 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d82ebb78584ea8a3789e70b270f127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731899707352 2024-11-18T03:15:08,749 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:08,749 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/f18506d7b4614f169329b15f6c6e374e is 50, key is test_row_0/B:col10/1731899707703/Put/seqid=0 2024-11-18T03:15:08,754 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:08,755 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/44dfaa6629154d3bb02976a9f97d61e5 is 50, key is test_row_0/A:col10/1731899707703/Put/seqid=0 2024-11-18T03:15:08,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741925_1101 (size=13289) 2024-11-18T03:15:08,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741924_1100 (size=13289) 2024-11-18T03:15:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T03:15:08,797 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/f18506d7b4614f169329b15f6c6e374e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f18506d7b4614f169329b15f6c6e374e 2024-11-18T03:15:08,808 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into f18506d7b4614f169329b15f6c6e374e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:08,808 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:08,808 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899708718; duration=0sec 2024-11-18T03:15:08,808 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:08,808 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:08,808 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:08,810 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:08,810 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:08,810 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:08,810 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6f5a2d81b6ee428998a923273c5716dd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b312adf5c9154be4b478d333ce787b5e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a4a638b85051402fb69144e9c09e269d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=36.9 K 2024-11-18T03:15:08,811 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5a2d81b6ee428998a923273c5716dd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899705099 2024-11-18T03:15:08,811 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b312adf5c9154be4b478d333ce787b5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731899706108 2024-11-18T03:15:08,812 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a638b85051402fb69144e9c09e269d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731899707352 2024-11-18T03:15:08,825 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:08,826 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/bf89d349a4b2449abab1490e54a6e1e3 is 50, key is test_row_0/C:col10/1731899707703/Put/seqid=0 2024-11-18T03:15:08,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741926_1102 (size=13289) 2024-11-18T03:15:08,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T03:15:08,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:08,840 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T03:15:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:08,844 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/bf89d349a4b2449abab1490e54a6e1e3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bf89d349a4b2449abab1490e54a6e1e3 2024-11-18T03:15:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/f298b26bbbe8414ab302a258c5eea32d is 50, key is test_row_0/A:col10/1731899707781/Put/seqid=0 2024-11-18T03:15:08,858 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into bf89d349a4b2449abab1490e54a6e1e3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:08,858 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:08,858 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899708718; duration=0sec 2024-11-18T03:15:08,858 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:08,858 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:08,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741927_1103 (size=12301) 2024-11-18T03:15:08,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/f298b26bbbe8414ab302a258c5eea32d 2024-11-18T03:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/86254d5b1d04425e891ce5ec87f3e63f is 50, key is test_row_0/B:col10/1731899707781/Put/seqid=0 2024-11-18T03:15:08,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741928_1104 (size=12301) 2024-11-18T03:15:08,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/86254d5b1d04425e891ce5ec87f3e63f 2024-11-18T03:15:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/2555121e9b03461685e4db0fa08f784e is 50, key is test_row_0/C:col10/1731899707781/Put/seqid=0 2024-11-18T03:15:08,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741929_1105 (size=12301) 2024-11-18T03:15:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:08,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:08,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899768958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899768958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899768960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899768961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:08,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899768971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T03:15:09,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899769071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899769071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899769081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,199 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/44dfaa6629154d3bb02976a9f97d61e5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/44dfaa6629154d3bb02976a9f97d61e5 2024-11-18T03:15:09,217 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 44dfaa6629154d3bb02976a9f97d61e5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:09,217 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:09,217 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899708717; duration=0sec 2024-11-18T03:15:09,218 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:09,218 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899769280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899769280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899769291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T03:15:09,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/2555121e9b03461685e4db0fa08f784e 2024-11-18T03:15:09,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/f298b26bbbe8414ab302a258c5eea32d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f298b26bbbe8414ab302a258c5eea32d 2024-11-18T03:15:09,357 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f298b26bbbe8414ab302a258c5eea32d, entries=150, sequenceid=423, filesize=12.0 K 2024-11-18T03:15:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/86254d5b1d04425e891ce5ec87f3e63f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/86254d5b1d04425e891ce5ec87f3e63f 2024-11-18T03:15:09,366 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/86254d5b1d04425e891ce5ec87f3e63f, entries=150, sequenceid=423, filesize=12.0 K 2024-11-18T03:15:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/2555121e9b03461685e4db0fa08f784e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/2555121e9b03461685e4db0fa08f784e 2024-11-18T03:15:09,374 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/2555121e9b03461685e4db0fa08f784e, entries=150, sequenceid=423, filesize=12.0 K 2024-11-18T03:15:09,375 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 29141263847cbdef9bad2179de1271a3 in 535ms, sequenceid=423, compaction requested=false 2024-11-18T03:15:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-18T03:15:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-18T03:15:09,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-18T03:15:09,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 692 msec 2024-11-18T03:15:09,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 703 msec 2024-11-18T03:15:09,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T03:15:09,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:09,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:09,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:09,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:09,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:09,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:09,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/9b3edc3b4e594b509b552fa710f97f9b is 50, key is test_row_0/A:col10/1731899708956/Put/seqid=0 2024-11-18T03:15:09,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741930_1106 (size=12301) 2024-11-18T03:15:09,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/9b3edc3b4e594b509b552fa710f97f9b 2024-11-18T03:15:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899769659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899769662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/196b8f04a0fc4fdeb72aa4a7c655ab0f is 50, key is test_row_0/B:col10/1731899708956/Put/seqid=0 2024-11-18T03:15:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899769666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741931_1107 (size=12301) 2024-11-18T03:15:09,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/196b8f04a0fc4fdeb72aa4a7c655ab0f 2024-11-18T03:15:09,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/053fe522d924400d8be46ace62d1d791 is 50, key is test_row_0/C:col10/1731899708956/Put/seqid=0 2024-11-18T03:15:09,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741932_1108 (size=12301) 2024-11-18T03:15:09,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/053fe522d924400d8be46ace62d1d791 2024-11-18T03:15:09,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/9b3edc3b4e594b509b552fa710f97f9b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/9b3edc3b4e594b509b552fa710f97f9b 2024-11-18T03:15:09,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/9b3edc3b4e594b509b552fa710f97f9b, entries=150, sequenceid=439, filesize=12.0 K 2024-11-18T03:15:09,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/196b8f04a0fc4fdeb72aa4a7c655ab0f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/196b8f04a0fc4fdeb72aa4a7c655ab0f 2024-11-18T03:15:09,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/196b8f04a0fc4fdeb72aa4a7c655ab0f, entries=150, sequenceid=439, filesize=12.0 K 2024-11-18T03:15:09,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/053fe522d924400d8be46ace62d1d791 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/053fe522d924400d8be46ace62d1d791 2024-11-18T03:15:09,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/053fe522d924400d8be46ace62d1d791, entries=150, sequenceid=439, filesize=12.0 K 2024-11-18T03:15:09,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 29141263847cbdef9bad2179de1271a3 in 176ms, sequenceid=439, compaction requested=true 2024-11-18T03:15:09,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:09,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:09,774 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:09,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:09,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:09,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:09,774 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:09,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:09,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:09,776 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:09,776 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:09,776 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:09,776 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:09,776 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:09,776 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:09,776 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/44dfaa6629154d3bb02976a9f97d61e5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f298b26bbbe8414ab302a258c5eea32d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/9b3edc3b4e594b509b552fa710f97f9b] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=37.0 K 2024-11-18T03:15:09,776 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f18506d7b4614f169329b15f6c6e374e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/86254d5b1d04425e891ce5ec87f3e63f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/196b8f04a0fc4fdeb72aa4a7c655ab0f] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=37.0 K 2024-11-18T03:15:09,777 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f18506d7b4614f169329b15f6c6e374e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731899707352 2024-11-18T03:15:09,777 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44dfaa6629154d3bb02976a9f97d61e5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731899707352 2024-11-18T03:15:09,777 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 86254d5b1d04425e891ce5ec87f3e63f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731899707767 2024-11-18T03:15:09,778 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f298b26bbbe8414ab302a258c5eea32d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731899707767 2024-11-18T03:15:09,778 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3edc3b4e594b509b552fa710f97f9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731899708956 2024-11-18T03:15:09,778 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 196b8f04a0fc4fdeb72aa4a7c655ab0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731899708956 2024-11-18T03:15:09,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T03:15:09,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:09,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:09,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:09,793 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:09,794 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3eb721519317468a87ceeec7e6ab8d2f is 50, key is test_row_0/A:col10/1731899708956/Put/seqid=0 2024-11-18T03:15:09,806 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:09,807 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/0f4edc29bb104491a0a2eaf0805f53c3 is 50, key is test_row_0/B:col10/1731899708956/Put/seqid=0 2024-11-18T03:15:09,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/c5a094e5e8bc42908004592681e20152 is 50, key is test_row_0/A:col10/1731899709649/Put/seqid=0 2024-11-18T03:15:09,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899769809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T03:15:09,821 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:09,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899769815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899769818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-18T03:15:09,824 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T03:15:09,825 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:09,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:09,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741934_1110 (size=13391) 2024-11-18T03:15:09,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741933_1109 (size=13391) 2024-11-18T03:15:09,874 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3eb721519317468a87ceeec7e6ab8d2f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3eb721519317468a87ceeec7e6ab8d2f 2024-11-18T03:15:09,874 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/0f4edc29bb104491a0a2eaf0805f53c3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0f4edc29bb104491a0a2eaf0805f53c3 2024-11-18T03:15:09,890 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 0f4edc29bb104491a0a2eaf0805f53c3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:09,890 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:09,890 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899709774; duration=0sec 2024-11-18T03:15:09,890 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:09,890 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:09,892 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 3eb721519317468a87ceeec7e6ab8d2f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:09,892 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:09,893 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899709774; duration=0sec 2024-11-18T03:15:09,893 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:09,893 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:09,893 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:09,895 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:09,895 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:09,895 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:09,895 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bf89d349a4b2449abab1490e54a6e1e3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/2555121e9b03461685e4db0fa08f784e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/053fe522d924400d8be46ace62d1d791] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=37.0 K 2024-11-18T03:15:09,896 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting bf89d349a4b2449abab1490e54a6e1e3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731899707352 2024-11-18T03:15:09,896 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 2555121e9b03461685e4db0fa08f784e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731899707767 2024-11-18T03:15:09,897 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 053fe522d924400d8be46ace62d1d791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731899708956 2024-11-18T03:15:09,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741935_1111 (size=14741) 2024-11-18T03:15:09,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/c5a094e5e8bc42908004592681e20152 2024-11-18T03:15:09,919 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:09,920 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/de19bf4cbb9444209e2f30b3e9387c4f is 50, key is test_row_0/C:col10/1731899708956/Put/seqid=0 2024-11-18T03:15:09,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/0e80ee6616474797a2e087f6b1544ba2 is 50, key is test_row_0/B:col10/1731899709649/Put/seqid=0 2024-11-18T03:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T03:15:09,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899769931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899769932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899769932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741936_1112 (size=13391) 2024-11-18T03:15:09,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T03:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:09,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:09,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741937_1113 (size=12301) 2024-11-18T03:15:09,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/0e80ee6616474797a2e087f6b1544ba2 2024-11-18T03:15:09,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899769986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:09,991 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 29141263847cbdef9bad2179de1271a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:09,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899769990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:09,994 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 29141263847cbdef9bad2179de1271a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:09,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:09,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/959021d5fe2d4c1dbb151183797f1b79 is 50, key is test_row_0/C:col10/1731899709649/Put/seqid=0 2024-11-18T03:15:10,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741938_1114 (size=12301) 2024-11-18T03:15:10,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/959021d5fe2d4c1dbb151183797f1b79 2024-11-18T03:15:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/c5a094e5e8bc42908004592681e20152 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/c5a094e5e8bc42908004592681e20152 2024-11-18T03:15:10,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/c5a094e5e8bc42908004592681e20152, entries=200, sequenceid=458, filesize=14.4 K 2024-11-18T03:15:10,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/0e80ee6616474797a2e087f6b1544ba2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0e80ee6616474797a2e087f6b1544ba2 2024-11-18T03:15:10,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0e80ee6616474797a2e087f6b1544ba2, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T03:15:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/959021d5fe2d4c1dbb151183797f1b79 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/959021d5fe2d4c1dbb151183797f1b79 2024-11-18T03:15:10,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/959021d5fe2d4c1dbb151183797f1b79, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T03:15:10,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 29141263847cbdef9bad2179de1271a3 in 290ms, sequenceid=458, compaction requested=false 2024-11-18T03:15:10,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T03:15:10,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T03:15:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,143 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:15:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2885d931bca04858ac0189cc11b1bfd8 is 50, key is test_row_0/A:col10/1731899709798/Put/seqid=0 2024-11-18T03:15:10,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899770172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899770173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899770175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741939_1115 (size=12301) 2024-11-18T03:15:10,186 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2885d931bca04858ac0189cc11b1bfd8 2024-11-18T03:15:10,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/1d24f63c979745bf96d60d9264a485c2 is 50, key is test_row_0/B:col10/1731899709798/Put/seqid=0 2024-11-18T03:15:10,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741940_1116 (size=12301) 2024-11-18T03:15:10,227 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/1d24f63c979745bf96d60d9264a485c2 2024-11-18T03:15:10,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/bc006ffab8644426b6e4f10e21fbc477 is 50, key is test_row_0/C:col10/1731899709798/Put/seqid=0 2024-11-18T03:15:10,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741941_1117 (size=12301) 2024-11-18T03:15:10,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/bc006ffab8644426b6e4f10e21fbc477 2024-11-18T03:15:10,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/2885d931bca04858ac0189cc11b1bfd8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2885d931bca04858ac0189cc11b1bfd8 2024-11-18T03:15:10,278 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2885d931bca04858ac0189cc11b1bfd8, entries=150, sequenceid=476, filesize=12.0 K 2024-11-18T03:15:10,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/1d24f63c979745bf96d60d9264a485c2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d24f63c979745bf96d60d9264a485c2 2024-11-18T03:15:10,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899770281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,286 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d24f63c979745bf96d60d9264a485c2, entries=150, sequenceid=476, filesize=12.0 K 2024-11-18T03:15:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899770281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899770281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/bc006ffab8644426b6e4f10e21fbc477 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bc006ffab8644426b6e4f10e21fbc477 2024-11-18T03:15:10,319 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bc006ffab8644426b6e4f10e21fbc477, entries=150, sequenceid=476, filesize=12.0 K 2024-11-18T03:15:10,320 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 29141263847cbdef9bad2179de1271a3 in 177ms, sequenceid=476, compaction requested=true 2024-11-18T03:15:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-18T03:15:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-18T03:15:10,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T03:15:10,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 497 msec 2024-11-18T03:15:10,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 504 msec 2024-11-18T03:15:10,374 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/de19bf4cbb9444209e2f30b3e9387c4f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/de19bf4cbb9444209e2f30b3e9387c4f 2024-11-18T03:15:10,381 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into de19bf4cbb9444209e2f30b3e9387c4f(size=13.1 K), total size for store is 37.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:10,381 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:10,381 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899709774; duration=0sec 2024-11-18T03:15:10,381 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:10,381 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T03:15:10,451 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:10,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-18T03:15:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T03:15:10,458 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:10,459 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:10,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:10,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:15:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:10,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:10,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:10,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:10,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3e4b89b2214d409da4aaa28a603a59c9 is 50, key is test_row_0/A:col10/1731899710495/Put/seqid=0 2024-11-18T03:15:10,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741942_1118 (size=12301) 2024-11-18T03:15:10,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3e4b89b2214d409da4aaa28a603a59c9 2024-11-18T03:15:10,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899770528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899770531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899770533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/3ebbed274e964dde9be3ebbf323b8532 is 50, key is test_row_0/B:col10/1731899710495/Put/seqid=0 2024-11-18T03:15:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T03:15:10,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741943_1119 (size=12301) 2024-11-18T03:15:10,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T03:15:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899770641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899770641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899770641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T03:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T03:15:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899770851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899770851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899770851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:10,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T03:15:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:10,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:10,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/3ebbed274e964dde9be3ebbf323b8532 2024-11-18T03:15:10,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b97f7fc13fb642188160bdeacf731973 is 50, key is test_row_0/C:col10/1731899710495/Put/seqid=0 2024-11-18T03:15:11,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741944_1120 (size=12301) 2024-11-18T03:15:11,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b97f7fc13fb642188160bdeacf731973 2024-11-18T03:15:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/3e4b89b2214d409da4aaa28a603a59c9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3e4b89b2214d409da4aaa28a603a59c9 2024-11-18T03:15:11,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3e4b89b2214d409da4aaa28a603a59c9, entries=150, sequenceid=494, filesize=12.0 K 2024-11-18T03:15:11,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/3ebbed274e964dde9be3ebbf323b8532 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3ebbed274e964dde9be3ebbf323b8532 2024-11-18T03:15:11,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3ebbed274e964dde9be3ebbf323b8532, entries=150, sequenceid=494, filesize=12.0 K 2024-11-18T03:15:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/b97f7fc13fb642188160bdeacf731973 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b97f7fc13fb642188160bdeacf731973 2024-11-18T03:15:11,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b97f7fc13fb642188160bdeacf731973, entries=150, sequenceid=494, filesize=12.0 K 2024-11-18T03:15:11,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 29141263847cbdef9bad2179de1271a3 in 567ms, sequenceid=494, compaction requested=true 2024-11-18T03:15:11,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:11,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:11,064 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:11,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:11,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:11,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:11,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:11,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:15:11,064 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:11,065 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:11,066 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:11,066 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:11,066 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3eb721519317468a87ceeec7e6ab8d2f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/c5a094e5e8bc42908004592681e20152, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2885d931bca04858ac0189cc11b1bfd8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3e4b89b2214d409da4aaa28a603a59c9] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=51.5 K 2024-11-18T03:15:11,066 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:11,066 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:11,066 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:11,067 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0f4edc29bb104491a0a2eaf0805f53c3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0e80ee6616474797a2e087f6b1544ba2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d24f63c979745bf96d60d9264a485c2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3ebbed274e964dde9be3ebbf323b8532] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=49.1 K 2024-11-18T03:15:11,071 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4edc29bb104491a0a2eaf0805f53c3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731899708956 2024-11-18T03:15:11,071 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb721519317468a87ceeec7e6ab8d2f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731899708956 2024-11-18T03:15:11,072 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e80ee6616474797a2e087f6b1544ba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731899709649 2024-11-18T03:15:11,073 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a094e5e8bc42908004592681e20152, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731899709649 2024-11-18T03:15:11,073 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d24f63c979745bf96d60d9264a485c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731899709798 2024-11-18T03:15:11,077 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2885d931bca04858ac0189cc11b1bfd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731899709798 2024-11-18T03:15:11,077 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ebbed274e964dde9be3ebbf323b8532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731899710171 2024-11-18T03:15:11,078 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4b89b2214d409da4aaa28a603a59c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731899710171 2024-11-18T03:15:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T03:15:11,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T03:15:11,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:11,098 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:11,102 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:11,103 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/892acd28760a4a178e5a57b013b20e66 is 50, key is test_row_0/A:col10/1731899710495/Put/seqid=0 2024-11-18T03:15:11,115 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:11,115 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/39c1810942734faead38258972734576 is 50, key is test_row_0/B:col10/1731899710495/Put/seqid=0 2024-11-18T03:15:11,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/62714e67ea204944ae84a1cf29808d3a is 50, key is test_row_1/A:col10/1731899710528/Put/seqid=0 2024-11-18T03:15:11,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741945_1121 (size=13527) 2024-11-18T03:15:11,157 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/892acd28760a4a178e5a57b013b20e66 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/892acd28760a4a178e5a57b013b20e66 2024-11-18T03:15:11,163 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 892acd28760a4a178e5a57b013b20e66(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:11,163 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:11,163 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=12, startTime=1731899711063; duration=0sec 2024-11-18T03:15:11,164 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:11,164 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:11,164 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:15:11,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741946_1122 (size=13527) 2024-11-18T03:15:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:11,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. as already flushing 2024-11-18T03:15:11,172 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:15:11,172 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:11,173 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:11,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741947_1123 (size=9857) 2024-11-18T03:15:11,179 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/39c1810942734faead38258972734576 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/39c1810942734faead38258972734576 2024-11-18T03:15:11,179 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/de19bf4cbb9444209e2f30b3e9387c4f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/959021d5fe2d4c1dbb151183797f1b79, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bc006ffab8644426b6e4f10e21fbc477, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b97f7fc13fb642188160bdeacf731973] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=49.1 K 2024-11-18T03:15:11,180 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting de19bf4cbb9444209e2f30b3e9387c4f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731899708956 2024-11-18T03:15:11,180 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/62714e67ea204944ae84a1cf29808d3a 2024-11-18T03:15:11,181 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959021d5fe2d4c1dbb151183797f1b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731899709649 2024-11-18T03:15:11,183 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc006ffab8644426b6e4f10e21fbc477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731899709798 2024-11-18T03:15:11,186 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting b97f7fc13fb642188160bdeacf731973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731899710171 2024-11-18T03:15:11,195 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into 39c1810942734faead38258972734576(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:11,195 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:11,195 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=12, startTime=1731899711064; duration=0sec 2024-11-18T03:15:11,195 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:11,195 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:11,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/8e70ec1e78114f8c80c37ed0624855c2 is 50, key is test_row_1/B:col10/1731899710528/Put/seqid=0 2024-11-18T03:15:11,208 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:11,208 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d46f0d424bf24fb5b2dfb937da28c63b is 50, key is test_row_0/C:col10/1731899710495/Put/seqid=0 2024-11-18T03:15:11,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899771235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741948_1124 (size=9857) 2024-11-18T03:15:11,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899771237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899771239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741949_1125 (size=13527) 2024-11-18T03:15:11,290 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/d46f0d424bf24fb5b2dfb937da28c63b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d46f0d424bf24fb5b2dfb937da28c63b 2024-11-18T03:15:11,308 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into d46f0d424bf24fb5b2dfb937da28c63b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:11,308 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:11,308 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=12, startTime=1731899711064; duration=0sec 2024-11-18T03:15:11,308 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:11,308 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899771351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899771351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899771351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899771561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899771561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899771570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T03:15:11,618 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-18T03:15:11,618 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:11,619 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-18T03:15:11,620 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:11,621 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-18T03:15:11,621 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-18T03:15:11,621 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:11,621 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:11,624 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,624 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,624 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,624 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,624 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,624 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,624 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:11,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:11,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:11,625 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:11,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:11,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/8e70ec1e78114f8c80c37ed0624855c2 2024-11-18T03:15:11,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9aa722a4f6d242d2ad9ef11850d69ebb is 50, key is test_row_1/C:col10/1731899710528/Put/seqid=0 2024-11-18T03:15:11,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741950_1126 (size=9857) 2024-11-18T03:15:11,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1731899771871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53832 deadline: 1731899771871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:11,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53806 deadline: 1731899771881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53794 deadline: 1731899772012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53818 deadline: 1731899772011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:12,012 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 29141263847cbdef9bad2179de1271a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:12,012 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 29141263847cbdef9bad2179de1271a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:12,064 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9aa722a4f6d242d2ad9ef11850d69ebb 2024-11-18T03:15:12,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/62714e67ea204944ae84a1cf29808d3a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/62714e67ea204944ae84a1cf29808d3a 2024-11-18T03:15:12,075 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/62714e67ea204944ae84a1cf29808d3a, entries=100, sequenceid=509, filesize=9.6 K 2024-11-18T03:15:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/8e70ec1e78114f8c80c37ed0624855c2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/8e70ec1e78114f8c80c37ed0624855c2 2024-11-18T03:15:12,081 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/8e70ec1e78114f8c80c37ed0624855c2, entries=100, sequenceid=509, filesize=9.6 K 2024-11-18T03:15:12,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/9aa722a4f6d242d2ad9ef11850d69ebb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9aa722a4f6d242d2ad9ef11850d69ebb 2024-11-18T03:15:12,086 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9aa722a4f6d242d2ad9ef11850d69ebb, entries=100, sequenceid=509, filesize=9.6 K 2024-11-18T03:15:12,087 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 29141263847cbdef9bad2179de1271a3 in 989ms, sequenceid=509, compaction requested=false 2024-11-18T03:15:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-18T03:15:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-18T03:15:12,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T03:15:12,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6300 sec 2024-11-18T03:15:12,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.6400 sec 2024-11-18T03:15:12,216 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T03:15:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:12,383 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-18T03:15:12,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T03:15:12,383 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:12,383 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-18T03:15:12,383 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:12,383 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:12,383 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:12,383 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:12,383 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:12,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:12,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:12,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:12,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:12,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:12,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:12,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:12,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:12,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebe9d8e06ea3455e8fd1e7e31e3e0346 is 50, key is test_row_0/A:col10/1731899712381/Put/seqid=0 2024-11-18T03:15:12,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741951_1127 (size=12301) 2024-11-18T03:15:12,392 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-18T03:15:12,392 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:12,392 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:12,392 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:12,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T03:15:12,602 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:12,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebe9d8e06ea3455e8fd1e7e31e3e0346 2024-11-18T03:15:12,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/a4b38b91a8e949679bf9268a00ef5cb7 is 50, key is test_row_0/B:col10/1731899712381/Put/seqid=0 2024-11-18T03:15:12,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741952_1128 (size=12301) 2024-11-18T03:15:13,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/a4b38b91a8e949679bf9268a00ef5cb7 2024-11-18T03:15:13,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/365b968679c34cb8b225d8d79cdf498f is 50, key is test_row_0/C:col10/1731899712381/Put/seqid=0 2024-11-18T03:15:13,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741953_1129 (size=12301) 2024-11-18T03:15:13,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/365b968679c34cb8b225d8d79cdf498f 2024-11-18T03:15:13,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebe9d8e06ea3455e8fd1e7e31e3e0346 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebe9d8e06ea3455e8fd1e7e31e3e0346 2024-11-18T03:15:13,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebe9d8e06ea3455e8fd1e7e31e3e0346, entries=150, sequenceid=532, filesize=12.0 K 2024-11-18T03:15:13,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/a4b38b91a8e949679bf9268a00ef5cb7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a4b38b91a8e949679bf9268a00ef5cb7 2024-11-18T03:15:13,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a4b38b91a8e949679bf9268a00ef5cb7, entries=150, sequenceid=532, filesize=12.0 K 2024-11-18T03:15:13,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/365b968679c34cb8b225d8d79cdf498f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/365b968679c34cb8b225d8d79cdf498f 2024-11-18T03:15:13,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/365b968679c34cb8b225d8d79cdf498f, entries=150, sequenceid=532, filesize=12.0 K 2024-11-18T03:15:13,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 29141263847cbdef9bad2179de1271a3 in 1276ms, sequenceid=532, compaction requested=true 2024-11-18T03:15:13,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:13,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:13,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:13,659 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:13,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:13,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:13,659 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:13,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29141263847cbdef9bad2179de1271a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:13,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:13,660 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:13,660 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:13,660 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/A is initiating minor compaction (all files) 2024-11-18T03:15:13,660 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/B is initiating minor compaction (all files) 2024-11-18T03:15:13,660 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/A in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:13,660 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/B in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:13,660 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/892acd28760a4a178e5a57b013b20e66, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/62714e67ea204944ae84a1cf29808d3a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebe9d8e06ea3455e8fd1e7e31e3e0346] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=34.8 K 2024-11-18T03:15:13,660 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/39c1810942734faead38258972734576, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/8e70ec1e78114f8c80c37ed0624855c2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a4b38b91a8e949679bf9268a00ef5cb7] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=34.8 K 2024-11-18T03:15:13,661 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892acd28760a4a178e5a57b013b20e66, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731899710171 2024-11-18T03:15:13,661 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c1810942734faead38258972734576, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731899710171 2024-11-18T03:15:13,661 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62714e67ea204944ae84a1cf29808d3a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731899710528 2024-11-18T03:15:13,661 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e70ec1e78114f8c80c37ed0624855c2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731899710528 2024-11-18T03:15:13,662 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b38b91a8e949679bf9268a00ef5cb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731899711190 2024-11-18T03:15:13,662 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe9d8e06ea3455e8fd1e7e31e3e0346, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731899711190 2024-11-18T03:15:13,670 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#A#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:13,670 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/86b0f805980748fa9efe9b4c589c2392 is 50, key is test_row_0/A:col10/1731899712381/Put/seqid=0 2024-11-18T03:15:13,678 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:13,679 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/c98bd1dab72e42f5baeea5e22ba167a3 is 50, key is test_row_0/B:col10/1731899712381/Put/seqid=0 2024-11-18T03:15:13,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741954_1130 (size=13629) 2024-11-18T03:15:13,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741955_1131 (size=13629) 2024-11-18T03:15:13,690 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/86b0f805980748fa9efe9b4c589c2392 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/86b0f805980748fa9efe9b4c589c2392 2024-11-18T03:15:13,697 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/A of 29141263847cbdef9bad2179de1271a3 into 86b0f805980748fa9efe9b4c589c2392(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:13,697 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:13,697 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/A, priority=13, startTime=1731899713659; duration=0sec 2024-11-18T03:15:13,697 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:13,697 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:A 2024-11-18T03:15:13,697 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:13,698 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:13,698 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 29141263847cbdef9bad2179de1271a3/C is initiating minor compaction (all files) 2024-11-18T03:15:13,699 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29141263847cbdef9bad2179de1271a3/C in TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:13,699 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d46f0d424bf24fb5b2dfb937da28c63b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9aa722a4f6d242d2ad9ef11850d69ebb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/365b968679c34cb8b225d8d79cdf498f] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp, totalSize=34.8 K 2024-11-18T03:15:13,699 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d46f0d424bf24fb5b2dfb937da28c63b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731899710171 2024-11-18T03:15:13,699 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aa722a4f6d242d2ad9ef11850d69ebb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731899710528 2024-11-18T03:15:13,700 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 365b968679c34cb8b225d8d79cdf498f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731899711190 2024-11-18T03:15:13,709 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29141263847cbdef9bad2179de1271a3#C#compaction#119 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:13,710 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/0f9363015bea40bea74fdb23a9582403 is 50, key is test_row_0/C:col10/1731899712381/Put/seqid=0 2024-11-18T03:15:13,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741956_1132 (size=13629) 2024-11-18T03:15:14,098 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/c98bd1dab72e42f5baeea5e22ba167a3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c98bd1dab72e42f5baeea5e22ba167a3 2024-11-18T03:15:14,105 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/B of 29141263847cbdef9bad2179de1271a3 into c98bd1dab72e42f5baeea5e22ba167a3(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:14,105 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:14,105 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/B, priority=13, startTime=1731899713659; duration=0sec 2024-11-18T03:15:14,105 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:14,105 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:B 2024-11-18T03:15:14,124 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/0f9363015bea40bea74fdb23a9582403 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/0f9363015bea40bea74fdb23a9582403 2024-11-18T03:15:14,131 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29141263847cbdef9bad2179de1271a3/C of 29141263847cbdef9bad2179de1271a3 into 0f9363015bea40bea74fdb23a9582403(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:14,131 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29141263847cbdef9bad2179de1271a3: 2024-11-18T03:15:14,131 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., storeName=29141263847cbdef9bad2179de1271a3/C, priority=13, startTime=1731899713659; duration=0sec 2024-11-18T03:15:14,131 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:14,131 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29141263847cbdef9bad2179de1271a3:C 2024-11-18T03:15:16,033 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-18T03:15:16,033 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:16,033 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:16,033 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:16,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:16,043 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-18T03:15:16,043 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:16,043 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:16,043 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:16,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2072 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2051 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 976 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2927 rows 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-18T03:15:16,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2931 rows 2024-11-18T03:15:16,044 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:15:16,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:15:16,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:16,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:16,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:16,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-18T03:15:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:16,059 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899716059"}]},"ts":"1731899716059"} 2024-11-18T03:15:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T03:15:16,062 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T03:15:16,062 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T03:15:16,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:15:16,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, UNASSIGN}] 2024-11-18T03:15:16,070 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, UNASSIGN 2024-11-18T03:15:16,071 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=29141263847cbdef9bad2179de1271a3, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:16,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, UNASSIGN because future has completed 2024-11-18T03:15:16,074 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:15:16,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:15:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T03:15:16,231 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:16,231 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 29141263847cbdef9bad2179de1271a3, disabling compactions & flushes 2024-11-18T03:15:16,232 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. after waiting 0 ms 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:16,232 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 29141263847cbdef9bad2179de1271a3 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=A 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=B 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:16,232 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29141263847cbdef9bad2179de1271a3, store=C 2024-11-18T03:15:16,233 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:16,237 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebb6958395bf4a5bbdf40cc43a2c0578 is 50, key is test_row_0/A:col10/1731899716031/Put/seqid=0 2024-11-18T03:15:16,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741957_1133 (size=9857) 2024-11-18T03:15:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T03:15:16,643 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebb6958395bf4a5bbdf40cc43a2c0578 2024-11-18T03:15:16,656 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/683d635b28e94bbe909f9cc11917c237 is 50, key is test_row_0/B:col10/1731899716031/Put/seqid=0 2024-11-18T03:15:16,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741958_1134 (size=9857) 2024-11-18T03:15:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T03:15:17,063 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/683d635b28e94bbe909f9cc11917c237 2024-11-18T03:15:17,075 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/eb9cc2ab765e4733bc2c3e6cc411b56d is 50, key is test_row_0/C:col10/1731899716031/Put/seqid=0 2024-11-18T03:15:17,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741959_1135 (size=9857) 2024-11-18T03:15:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T03:15:17,480 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/eb9cc2ab765e4733bc2c3e6cc411b56d 2024-11-18T03:15:17,485 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/A/ebb6958395bf4a5bbdf40cc43a2c0578 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebb6958395bf4a5bbdf40cc43a2c0578 2024-11-18T03:15:17,489 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebb6958395bf4a5bbdf40cc43a2c0578, entries=100, sequenceid=541, filesize=9.6 K 2024-11-18T03:15:17,490 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/B/683d635b28e94bbe909f9cc11917c237 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/683d635b28e94bbe909f9cc11917c237 2024-11-18T03:15:17,494 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/683d635b28e94bbe909f9cc11917c237, entries=100, sequenceid=541, filesize=9.6 K 2024-11-18T03:15:17,495 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/.tmp/C/eb9cc2ab765e4733bc2c3e6cc411b56d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/eb9cc2ab765e4733bc2c3e6cc411b56d 2024-11-18T03:15:17,500 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/eb9cc2ab765e4733bc2c3e6cc411b56d, entries=100, sequenceid=541, filesize=9.6 K 2024-11-18T03:15:17,501 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 29141263847cbdef9bad2179de1271a3 in 1269ms, sequenceid=541, compaction requested=false 2024-11-18T03:15:17,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d68bd7437c37472cb77e3af39d635850, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e1e9b0a8a9b7478e8d3581836573be85, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebf7a3480a774095ba21090adcd32bdb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3f0135fb6c994311a136980c84384779, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7c7c9153f60e48348b8347abf2e948e0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/472935e907a745e08c67d37479aacbba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/eef21d36af404c02bc666f74f99719d6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/efc7376868d4493687e27a4fbecb6668, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a4c1ac5e9e004dc8bc6ec54a5605370a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/80afcf6978374a5a9f6bb22649893059, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e466dadf26024f35a0a904d9467dd018, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2339e197b61d456ea4cd49696f5c4c7f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2ffd7ae071ca46f9bcd2489d34a79923, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/73e29039d9474894b341e283cee7ccb8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a16cc9eceae2437e965a56aeecc2d961, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6067a57fa2b54cb8bd44cdbc13b77f39, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/96866764676e4885a622c0b103933a26, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23421a43b37c451094b5143de16ad359, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d271e25f84504a0484df64b8431126e5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6157d9d6cc1f40c18945abf56ea9eb70, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/006b9c9f6ea143a0b8df947e1a3cd0af, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a07e9561c47a4be4bbc378828e2b724c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3ce2f6a4befa4af8b9f9cf55f71487bb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23ca083a5a8b4938a94d9c64afcfffcd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/cd3d2cbf463a4679b480153574149d8e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a6f224c59bc64d70aeec1676f9daae32, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/8329aeb1adcb48efb6bf0a97cabdfaf1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/44dfaa6629154d3bb02976a9f97d61e5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f298b26bbbe8414ab302a258c5eea32d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3eb721519317468a87ceeec7e6ab8d2f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/9b3edc3b4e594b509b552fa710f97f9b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/c5a094e5e8bc42908004592681e20152, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2885d931bca04858ac0189cc11b1bfd8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/892acd28760a4a178e5a57b013b20e66, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3e4b89b2214d409da4aaa28a603a59c9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/62714e67ea204944ae84a1cf29808d3a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebe9d8e06ea3455e8fd1e7e31e3e0346] to archive 2024-11-18T03:15:17,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:15:17,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d68bd7437c37472cb77e3af39d635850 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d68bd7437c37472cb77e3af39d635850 2024-11-18T03:15:17,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e1e9b0a8a9b7478e8d3581836573be85 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e1e9b0a8a9b7478e8d3581836573be85 2024-11-18T03:15:17,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebf7a3480a774095ba21090adcd32bdb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebf7a3480a774095ba21090adcd32bdb 2024-11-18T03:15:17,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7a0e62f5f75f4ff5b2b52ad9a1d9bf91 2024-11-18T03:15:17,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3f0135fb6c994311a136980c84384779 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3f0135fb6c994311a136980c84384779 2024-11-18T03:15:17,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7c7c9153f60e48348b8347abf2e948e0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/7c7c9153f60e48348b8347abf2e948e0 2024-11-18T03:15:17,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/472935e907a745e08c67d37479aacbba to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/472935e907a745e08c67d37479aacbba 2024-11-18T03:15:17,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/eef21d36af404c02bc666f74f99719d6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/eef21d36af404c02bc666f74f99719d6 2024-11-18T03:15:17,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/efc7376868d4493687e27a4fbecb6668 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/efc7376868d4493687e27a4fbecb6668 2024-11-18T03:15:17,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a4c1ac5e9e004dc8bc6ec54a5605370a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a4c1ac5e9e004dc8bc6ec54a5605370a 2024-11-18T03:15:17,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/80afcf6978374a5a9f6bb22649893059 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/80afcf6978374a5a9f6bb22649893059 2024-11-18T03:15:17,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e466dadf26024f35a0a904d9467dd018 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/e466dadf26024f35a0a904d9467dd018 2024-11-18T03:15:17,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2339e197b61d456ea4cd49696f5c4c7f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2339e197b61d456ea4cd49696f5c4c7f 2024-11-18T03:15:17,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2ffd7ae071ca46f9bcd2489d34a79923 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2ffd7ae071ca46f9bcd2489d34a79923 2024-11-18T03:15:17,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f7e2d06bfc4e4cf0853eab9b41b8ff8d 2024-11-18T03:15:17,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/73e29039d9474894b341e283cee7ccb8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/73e29039d9474894b341e283cee7ccb8 2024-11-18T03:15:17,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a16cc9eceae2437e965a56aeecc2d961 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a16cc9eceae2437e965a56aeecc2d961 2024-11-18T03:15:17,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6067a57fa2b54cb8bd44cdbc13b77f39 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6067a57fa2b54cb8bd44cdbc13b77f39 2024-11-18T03:15:17,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/96866764676e4885a622c0b103933a26 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/96866764676e4885a622c0b103933a26 2024-11-18T03:15:17,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23421a43b37c451094b5143de16ad359 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23421a43b37c451094b5143de16ad359 2024-11-18T03:15:17,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d271e25f84504a0484df64b8431126e5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/d271e25f84504a0484df64b8431126e5 2024-11-18T03:15:17,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6157d9d6cc1f40c18945abf56ea9eb70 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/6157d9d6cc1f40c18945abf56ea9eb70 2024-11-18T03:15:17,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/006b9c9f6ea143a0b8df947e1a3cd0af to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/006b9c9f6ea143a0b8df947e1a3cd0af 2024-11-18T03:15:17,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a07e9561c47a4be4bbc378828e2b724c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a07e9561c47a4be4bbc378828e2b724c 2024-11-18T03:15:17,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3ce2f6a4befa4af8b9f9cf55f71487bb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3ce2f6a4befa4af8b9f9cf55f71487bb 2024-11-18T03:15:17,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23ca083a5a8b4938a94d9c64afcfffcd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/23ca083a5a8b4938a94d9c64afcfffcd 2024-11-18T03:15:17,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/cd3d2cbf463a4679b480153574149d8e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/cd3d2cbf463a4679b480153574149d8e 2024-11-18T03:15:17,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a6f224c59bc64d70aeec1676f9daae32 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/a6f224c59bc64d70aeec1676f9daae32 2024-11-18T03:15:17,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/8329aeb1adcb48efb6bf0a97cabdfaf1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/8329aeb1adcb48efb6bf0a97cabdfaf1 2024-11-18T03:15:17,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/44dfaa6629154d3bb02976a9f97d61e5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/44dfaa6629154d3bb02976a9f97d61e5 2024-11-18T03:15:17,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f298b26bbbe8414ab302a258c5eea32d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/f298b26bbbe8414ab302a258c5eea32d 2024-11-18T03:15:17,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3eb721519317468a87ceeec7e6ab8d2f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3eb721519317468a87ceeec7e6ab8d2f 2024-11-18T03:15:17,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/9b3edc3b4e594b509b552fa710f97f9b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/9b3edc3b4e594b509b552fa710f97f9b 2024-11-18T03:15:17,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/c5a094e5e8bc42908004592681e20152 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/c5a094e5e8bc42908004592681e20152 2024-11-18T03:15:17,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2885d931bca04858ac0189cc11b1bfd8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/2885d931bca04858ac0189cc11b1bfd8 2024-11-18T03:15:17,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/892acd28760a4a178e5a57b013b20e66 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/892acd28760a4a178e5a57b013b20e66 2024-11-18T03:15:17,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3e4b89b2214d409da4aaa28a603a59c9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/3e4b89b2214d409da4aaa28a603a59c9 2024-11-18T03:15:17,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/62714e67ea204944ae84a1cf29808d3a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/62714e67ea204944ae84a1cf29808d3a 2024-11-18T03:15:17,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebe9d8e06ea3455e8fd1e7e31e3e0346 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebe9d8e06ea3455e8fd1e7e31e3e0346 2024-11-18T03:15:17,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7bcf8b878e6b4936a8719f6599da2bb6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/607314d38c694b1ba73f72c61232a116, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7a8c6e3db1f74356bcf09ce4870768c7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/96f91653d6f9479e90e9e1259c75ab98, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/358068d6b611498b8e7553dbaf08e045, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c9ab43e455d54b24ab07c8af751071d4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ab00b3a0aa3a426c86a7013a80304f37, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/68098dd21d554638841288becb72daa2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b855b8dfd542454c8a7ca6aab7e6185b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/4c9b1bcbb2c241bc9132dc4904ddaf86, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/17963c82defe4f5ca946ff9ce673497b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/57c505a808894099b9de72dfdd8a4f96, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3655cef67e834dcb8cfdc5822b506cb6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/eeb0d115222a4c28b1726ee367a5d7f2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/cb97d505869341f2ae6c08930afaf594, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7410d835eac14187be3a186f0943a920, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f6c1d9be00e44647bbd4a82aa1e795bb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/dab91974521449d9b09aa7001f8887cd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7c6493b9c0a84bf5ba08928e0f0dd6e3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ef2afe9dfcef488db159d4350b8b16a5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b74e3fe3598a4fae9aa0ee4600607e0d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/02b9740feb144922bc592352e1362742, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a5a72fce4e1c47d98d8405fd2e8c13ed, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/26c31026274c40d591adda31d15dd878, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e40cafa0b8614024894458e372ec5d7e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/35bfa4e2f11a4848a67e5ac41f8517fb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e1cb665ea8ac4bf18b6f273253859c97, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d94b38041b04c1997ac4f5c313e1d61, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f18506d7b4614f169329b15f6c6e374e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/47d82ebb78584ea8a3789e70b270f127, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/86254d5b1d04425e891ce5ec87f3e63f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0f4edc29bb104491a0a2eaf0805f53c3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/196b8f04a0fc4fdeb72aa4a7c655ab0f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0e80ee6616474797a2e087f6b1544ba2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d24f63c979745bf96d60d9264a485c2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/39c1810942734faead38258972734576, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3ebbed274e964dde9be3ebbf323b8532, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/8e70ec1e78114f8c80c37ed0624855c2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a4b38b91a8e949679bf9268a00ef5cb7] to archive 2024-11-18T03:15:17,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:15:17,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7bcf8b878e6b4936a8719f6599da2bb6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7bcf8b878e6b4936a8719f6599da2bb6 2024-11-18T03:15:17,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/607314d38c694b1ba73f72c61232a116 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/607314d38c694b1ba73f72c61232a116 2024-11-18T03:15:17,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7a8c6e3db1f74356bcf09ce4870768c7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7a8c6e3db1f74356bcf09ce4870768c7 2024-11-18T03:15:17,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/96f91653d6f9479e90e9e1259c75ab98 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/96f91653d6f9479e90e9e1259c75ab98 2024-11-18T03:15:17,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/358068d6b611498b8e7553dbaf08e045 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/358068d6b611498b8e7553dbaf08e045 2024-11-18T03:15:17,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c9ab43e455d54b24ab07c8af751071d4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c9ab43e455d54b24ab07c8af751071d4 2024-11-18T03:15:17,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ab00b3a0aa3a426c86a7013a80304f37 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ab00b3a0aa3a426c86a7013a80304f37 2024-11-18T03:15:17,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/68098dd21d554638841288becb72daa2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/68098dd21d554638841288becb72daa2 2024-11-18T03:15:17,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b855b8dfd542454c8a7ca6aab7e6185b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b855b8dfd542454c8a7ca6aab7e6185b 2024-11-18T03:15:17,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/4c9b1bcbb2c241bc9132dc4904ddaf86 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/4c9b1bcbb2c241bc9132dc4904ddaf86 2024-11-18T03:15:17,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/17963c82defe4f5ca946ff9ce673497b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/17963c82defe4f5ca946ff9ce673497b 2024-11-18T03:15:17,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/57c505a808894099b9de72dfdd8a4f96 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/57c505a808894099b9de72dfdd8a4f96 2024-11-18T03:15:17,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3655cef67e834dcb8cfdc5822b506cb6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3655cef67e834dcb8cfdc5822b506cb6 2024-11-18T03:15:17,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/eeb0d115222a4c28b1726ee367a5d7f2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/eeb0d115222a4c28b1726ee367a5d7f2 2024-11-18T03:15:17,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/cb97d505869341f2ae6c08930afaf594 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/cb97d505869341f2ae6c08930afaf594 2024-11-18T03:15:17,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7410d835eac14187be3a186f0943a920 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7410d835eac14187be3a186f0943a920 2024-11-18T03:15:17,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f6c1d9be00e44647bbd4a82aa1e795bb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f6c1d9be00e44647bbd4a82aa1e795bb 2024-11-18T03:15:17,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/dab91974521449d9b09aa7001f8887cd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/dab91974521449d9b09aa7001f8887cd 2024-11-18T03:15:17,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7c6493b9c0a84bf5ba08928e0f0dd6e3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/7c6493b9c0a84bf5ba08928e0f0dd6e3 2024-11-18T03:15:17,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ef2afe9dfcef488db159d4350b8b16a5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/ef2afe9dfcef488db159d4350b8b16a5 2024-11-18T03:15:17,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b74e3fe3598a4fae9aa0ee4600607e0d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/b74e3fe3598a4fae9aa0ee4600607e0d 2024-11-18T03:15:17,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/02b9740feb144922bc592352e1362742 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/02b9740feb144922bc592352e1362742 2024-11-18T03:15:17,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a5a72fce4e1c47d98d8405fd2e8c13ed to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a5a72fce4e1c47d98d8405fd2e8c13ed 2024-11-18T03:15:17,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/26c31026274c40d591adda31d15dd878 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/26c31026274c40d591adda31d15dd878 2024-11-18T03:15:17,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e40cafa0b8614024894458e372ec5d7e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e40cafa0b8614024894458e372ec5d7e 2024-11-18T03:15:17,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/35bfa4e2f11a4848a67e5ac41f8517fb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/35bfa4e2f11a4848a67e5ac41f8517fb 2024-11-18T03:15:17,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e1cb665ea8ac4bf18b6f273253859c97 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/e1cb665ea8ac4bf18b6f273253859c97 2024-11-18T03:15:17,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d94b38041b04c1997ac4f5c313e1d61 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d94b38041b04c1997ac4f5c313e1d61 2024-11-18T03:15:17,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f18506d7b4614f169329b15f6c6e374e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/f18506d7b4614f169329b15f6c6e374e 2024-11-18T03:15:17,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/47d82ebb78584ea8a3789e70b270f127 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/47d82ebb78584ea8a3789e70b270f127 2024-11-18T03:15:17,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/86254d5b1d04425e891ce5ec87f3e63f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/86254d5b1d04425e891ce5ec87f3e63f 2024-11-18T03:15:17,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0f4edc29bb104491a0a2eaf0805f53c3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0f4edc29bb104491a0a2eaf0805f53c3 2024-11-18T03:15:17,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/196b8f04a0fc4fdeb72aa4a7c655ab0f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/196b8f04a0fc4fdeb72aa4a7c655ab0f 2024-11-18T03:15:17,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0e80ee6616474797a2e087f6b1544ba2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/0e80ee6616474797a2e087f6b1544ba2 2024-11-18T03:15:17,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d24f63c979745bf96d60d9264a485c2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/1d24f63c979745bf96d60d9264a485c2 2024-11-18T03:15:17,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/39c1810942734faead38258972734576 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/39c1810942734faead38258972734576 2024-11-18T03:15:17,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3ebbed274e964dde9be3ebbf323b8532 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/3ebbed274e964dde9be3ebbf323b8532 2024-11-18T03:15:17,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/8e70ec1e78114f8c80c37ed0624855c2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/8e70ec1e78114f8c80c37ed0624855c2 2024-11-18T03:15:17,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a4b38b91a8e949679bf9268a00ef5cb7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/a4b38b91a8e949679bf9268a00ef5cb7 2024-11-18T03:15:17,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/428ece1356ca475e9dfe0bff598f4888, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6ba52b6551c444eeb8e48bd670691bc9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/626e20aabb0d415baf17a060eeda4365, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/193e35daca4d40268df4e1c52805823f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/e94a709d22644250aab04692ca4510cf, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/192f764bebc646f79dfeb5ec51b5003f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/45063f8e87a04bc5b13197a1467cb32a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7e468e3d1b9945e191b1a581f383a023, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/91bdcf6b9cb349e9860a5c07caacc7e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/862c2296cd9c4496b875664813aabd1c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d907a8d7e4ab4a939b29637a3fb2585c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/031340de9a424b4284bd45e76c622029, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d310cdad4f664f9c81738610e4f020ae, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/ff222f0837cc4fb48d0734537ca42cd4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9e5c87aa42654a4482094483c40fd72d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/4ff0f2428f304ba4a17946618743576b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/866e63474b284ab6886eb1bc49ca9815, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b36ca46d4fe841cd8023fff9110be6a3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/678a1b254fb64e6193e9e8df34718e09, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/062787d7d84e4efba33cbd7c4bfa8fc8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a8d3b6cb5c01476a8ec1ecec45617b32, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/481e2742eb444d3d85a3a922340ab073, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/92906373dec84d84ab0f5170601ed0c7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9038e49fc7a446978791ae69be12c75e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7c58d7792de44384a39f665cab98ded3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6f5a2d81b6ee428998a923273c5716dd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a3b26e2c30254074a1b3451e8c0b4191, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b312adf5c9154be4b478d333ce787b5e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bf89d349a4b2449abab1490e54a6e1e3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a4a638b85051402fb69144e9c09e269d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/2555121e9b03461685e4db0fa08f784e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/de19bf4cbb9444209e2f30b3e9387c4f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/053fe522d924400d8be46ace62d1d791, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/959021d5fe2d4c1dbb151183797f1b79, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bc006ffab8644426b6e4f10e21fbc477, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d46f0d424bf24fb5b2dfb937da28c63b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b97f7fc13fb642188160bdeacf731973, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9aa722a4f6d242d2ad9ef11850d69ebb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/365b968679c34cb8b225d8d79cdf498f] to archive 2024-11-18T03:15:17,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:15:17,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/428ece1356ca475e9dfe0bff598f4888 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/428ece1356ca475e9dfe0bff598f4888 2024-11-18T03:15:17,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6ba52b6551c444eeb8e48bd670691bc9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6ba52b6551c444eeb8e48bd670691bc9 2024-11-18T03:15:17,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/626e20aabb0d415baf17a060eeda4365 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/626e20aabb0d415baf17a060eeda4365 2024-11-18T03:15:17,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/193e35daca4d40268df4e1c52805823f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/193e35daca4d40268df4e1c52805823f 2024-11-18T03:15:17,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/e94a709d22644250aab04692ca4510cf to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/e94a709d22644250aab04692ca4510cf 2024-11-18T03:15:17,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/192f764bebc646f79dfeb5ec51b5003f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/192f764bebc646f79dfeb5ec51b5003f 2024-11-18T03:15:17,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/45063f8e87a04bc5b13197a1467cb32a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/45063f8e87a04bc5b13197a1467cb32a 2024-11-18T03:15:17,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7e468e3d1b9945e191b1a581f383a023 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7e468e3d1b9945e191b1a581f383a023 2024-11-18T03:15:17,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/91bdcf6b9cb349e9860a5c07caacc7e2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/91bdcf6b9cb349e9860a5c07caacc7e2 2024-11-18T03:15:17,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/862c2296cd9c4496b875664813aabd1c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/862c2296cd9c4496b875664813aabd1c 2024-11-18T03:15:17,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d907a8d7e4ab4a939b29637a3fb2585c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d907a8d7e4ab4a939b29637a3fb2585c 2024-11-18T03:15:17,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/031340de9a424b4284bd45e76c622029 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/031340de9a424b4284bd45e76c622029 2024-11-18T03:15:17,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d310cdad4f664f9c81738610e4f020ae to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d310cdad4f664f9c81738610e4f020ae 2024-11-18T03:15:17,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/ff222f0837cc4fb48d0734537ca42cd4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/ff222f0837cc4fb48d0734537ca42cd4 2024-11-18T03:15:17,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9e5c87aa42654a4482094483c40fd72d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9e5c87aa42654a4482094483c40fd72d 2024-11-18T03:15:17,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/4ff0f2428f304ba4a17946618743576b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/4ff0f2428f304ba4a17946618743576b 2024-11-18T03:15:17,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/866e63474b284ab6886eb1bc49ca9815 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/866e63474b284ab6886eb1bc49ca9815 2024-11-18T03:15:17,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b36ca46d4fe841cd8023fff9110be6a3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b36ca46d4fe841cd8023fff9110be6a3 2024-11-18T03:15:17,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/678a1b254fb64e6193e9e8df34718e09 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/678a1b254fb64e6193e9e8df34718e09 2024-11-18T03:15:17,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/062787d7d84e4efba33cbd7c4bfa8fc8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/062787d7d84e4efba33cbd7c4bfa8fc8 2024-11-18T03:15:17,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a8d3b6cb5c01476a8ec1ecec45617b32 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a8d3b6cb5c01476a8ec1ecec45617b32 2024-11-18T03:15:17,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/481e2742eb444d3d85a3a922340ab073 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/481e2742eb444d3d85a3a922340ab073 2024-11-18T03:15:17,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/92906373dec84d84ab0f5170601ed0c7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/92906373dec84d84ab0f5170601ed0c7 2024-11-18T03:15:17,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9038e49fc7a446978791ae69be12c75e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9038e49fc7a446978791ae69be12c75e 2024-11-18T03:15:17,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7c58d7792de44384a39f665cab98ded3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/7c58d7792de44384a39f665cab98ded3 2024-11-18T03:15:17,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6f5a2d81b6ee428998a923273c5716dd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/6f5a2d81b6ee428998a923273c5716dd 2024-11-18T03:15:17,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a3b26e2c30254074a1b3451e8c0b4191 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a3b26e2c30254074a1b3451e8c0b4191 2024-11-18T03:15:17,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b312adf5c9154be4b478d333ce787b5e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b312adf5c9154be4b478d333ce787b5e 2024-11-18T03:15:17,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bf89d349a4b2449abab1490e54a6e1e3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bf89d349a4b2449abab1490e54a6e1e3 2024-11-18T03:15:17,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a4a638b85051402fb69144e9c09e269d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/a4a638b85051402fb69144e9c09e269d 2024-11-18T03:15:17,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/2555121e9b03461685e4db0fa08f784e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/2555121e9b03461685e4db0fa08f784e 2024-11-18T03:15:17,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/de19bf4cbb9444209e2f30b3e9387c4f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/de19bf4cbb9444209e2f30b3e9387c4f 2024-11-18T03:15:17,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/053fe522d924400d8be46ace62d1d791 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/053fe522d924400d8be46ace62d1d791 2024-11-18T03:15:17,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/959021d5fe2d4c1dbb151183797f1b79 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/959021d5fe2d4c1dbb151183797f1b79 2024-11-18T03:15:17,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bc006ffab8644426b6e4f10e21fbc477 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/bc006ffab8644426b6e4f10e21fbc477 2024-11-18T03:15:17,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d46f0d424bf24fb5b2dfb937da28c63b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/d46f0d424bf24fb5b2dfb937da28c63b 2024-11-18T03:15:17,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b97f7fc13fb642188160bdeacf731973 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/b97f7fc13fb642188160bdeacf731973 2024-11-18T03:15:17,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9aa722a4f6d242d2ad9ef11850d69ebb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/9aa722a4f6d242d2ad9ef11850d69ebb 2024-11-18T03:15:17,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/365b968679c34cb8b225d8d79cdf498f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/365b968679c34cb8b225d8d79cdf498f 2024-11-18T03:15:17,732 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/recovered.edits/544.seqid, newMaxSeqId=544, maxSeqId=1 2024-11-18T03:15:17,734 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. 2024-11-18T03:15:17,734 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 29141263847cbdef9bad2179de1271a3: Waiting for close lock at 1731899716231Running coprocessor pre-close hooks at 1731899716231Disabling compacts and flushes for region at 1731899716231Disabling writes for close at 1731899716232 (+1 ms)Obtaining lock to block concurrent updates at 1731899716232Preparing flush snapshotting stores in 29141263847cbdef9bad2179de1271a3 at 1731899716232Finished memstore snapshotting TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731899716233 (+1 ms)Flushing stores of TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3. at 1731899716233Flushing 29141263847cbdef9bad2179de1271a3/A: creating writer at 1731899716233Flushing 29141263847cbdef9bad2179de1271a3/A: appending metadata at 1731899716236 (+3 ms)Flushing 29141263847cbdef9bad2179de1271a3/A: closing flushed file at 1731899716236Flushing 29141263847cbdef9bad2179de1271a3/B: creating writer at 1731899716652 (+416 ms)Flushing 29141263847cbdef9bad2179de1271a3/B: appending metadata at 1731899716655 (+3 ms)Flushing 29141263847cbdef9bad2179de1271a3/B: closing flushed file at 1731899716655Flushing 29141263847cbdef9bad2179de1271a3/C: creating writer at 1731899717073 (+418 ms)Flushing 29141263847cbdef9bad2179de1271a3/C: appending metadata at 1731899717075 (+2 ms)Flushing 29141263847cbdef9bad2179de1271a3/C: closing flushed file at 1731899717075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a78dd83: reopening flushed file at 1731899717484 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e2a5fab: reopening flushed file at 1731899717489 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195b9b54: reopening flushed file at 1731899717494 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 29141263847cbdef9bad2179de1271a3 in 1269ms, sequenceid=541, compaction requested=false at 1731899717501 (+7 ms)Writing region close event to WAL at 1731899717725 (+224 ms)Running coprocessor post-close hooks at 1731899717734 (+9 ms)Closed at 1731899717734 2024-11-18T03:15:17,737 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:17,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=29141263847cbdef9bad2179de1271a3, regionState=CLOSED 2024-11-18T03:15:17,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:15:17,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-18T03:15:17,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 29141263847cbdef9bad2179de1271a3, server=fd862c3a04f9,38549,1731899684479 in 1.6680 sec 2024-11-18T03:15:17,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-18T03:15:17,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29141263847cbdef9bad2179de1271a3, UNASSIGN in 1.6750 sec 2024-11-18T03:15:17,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-18T03:15:17,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6830 sec 2024-11-18T03:15:17,750 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899717750"}]},"ts":"1731899717750"} 2024-11-18T03:15:17,752 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T03:15:17,752 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T03:15:17,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7010 sec 2024-11-18T03:15:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T03:15:18,211 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:18,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-18T03:15:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:18,219 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T03:15:18,221 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:18,226 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:18,231 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/recovered.edits] 2024-11-18T03:15:18,234 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/86b0f805980748fa9efe9b4c589c2392 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/86b0f805980748fa9efe9b4c589c2392 2024-11-18T03:15:18,240 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebb6958395bf4a5bbdf40cc43a2c0578 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/A/ebb6958395bf4a5bbdf40cc43a2c0578 2024-11-18T03:15:18,244 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/683d635b28e94bbe909f9cc11917c237 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/683d635b28e94bbe909f9cc11917c237 2024-11-18T03:15:18,245 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c98bd1dab72e42f5baeea5e22ba167a3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/B/c98bd1dab72e42f5baeea5e22ba167a3 2024-11-18T03:15:18,248 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/0f9363015bea40bea74fdb23a9582403 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/0f9363015bea40bea74fdb23a9582403 2024-11-18T03:15:18,250 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/eb9cc2ab765e4733bc2c3e6cc411b56d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/C/eb9cc2ab765e4733bc2c3e6cc411b56d 2024-11-18T03:15:18,252 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/recovered.edits/544.seqid to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3/recovered.edits/544.seqid 2024-11-18T03:15:18,253 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/29141263847cbdef9bad2179de1271a3 2024-11-18T03:15:18,253 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T03:15:18,260 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:18,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-18T03:15:18,273 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T03:15:18,312 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T03:15:18,318 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:18,318 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T03:15:18,318 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731899718318"}]},"ts":"9223372036854775807"} 2024-11-18T03:15:18,323 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T03:15:18,323 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 29141263847cbdef9bad2179de1271a3, NAME => 'TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T03:15:18,323 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T03:15:18,324 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731899718323"}]},"ts":"9223372036854775807"} 2024-11-18T03:15:18,327 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T03:15:18,328 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:18,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-11-18T03:15:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T03:15:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T03:15:18,331 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:18,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2140850966_22 at /127.0.0.1:51814 [Waiting for operation #375] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_114200576_22 at /127.0.0.1:36314 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2140850966_22 at /127.0.0.1:53050 [Waiting for operation #359] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_114200576_22 at /127.0.0.1:41002 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fd862c3a04f9:38549-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=871 (was 916), ProcessCount=11 (was 11), AvailableMemoryMB=3010 (was 2900) - AvailableMemoryMB LEAK? - 2024-11-18T03:15:18,353 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=871, ProcessCount=11, AvailableMemoryMB=3009 2024-11-18T03:15:18,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:15:18,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:15:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:18,357 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:15:18,357 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:18,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-18T03:15:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T03:15:18,358 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:15:18,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741960_1136 (size=1061) 2024-11-18T03:15:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T03:15:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T03:15:18,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:15:18,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741961_1137 (size=53) 2024-11-18T03:15:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T03:15:19,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:15:19,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f0d0b02e54813e28a58a04c19edb8553, disabling compactions & flushes 2024-11-18T03:15:19,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. after waiting 0 ms 2024-11-18T03:15:19,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0d0b02e54813e28a58a04c19edb8553: Waiting for close lock at 1731899719175Disabling compacts and flushes for region at 1731899719175Disabling writes for close at 1731899719176 (+1 ms)Writing region close event to WAL at 1731899719176Closed at 1731899719176 2024-11-18T03:15:19,177 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:15:19,177 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731899719177"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731899719177"}]},"ts":"1731899719177"} 2024-11-18T03:15:19,179 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:15:19,180 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:15:19,180 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899719180"}]},"ts":"1731899719180"} 2024-11-18T03:15:19,182 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T03:15:19,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, ASSIGN}] 2024-11-18T03:15:19,184 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, ASSIGN 2024-11-18T03:15:19,185 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, ASSIGN; state=OFFLINE, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=false 2024-11-18T03:15:19,336 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:19,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, ASSIGN because future has completed 2024-11-18T03:15:19,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:15:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T03:15:19,494 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,494 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:15:19,495 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,495 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:15:19,495 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,495 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,497 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,498 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:19,500 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0d0b02e54813e28a58a04c19edb8553 columnFamilyName A 2024-11-18T03:15:19,500 DEBUG [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:19,501 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(327): Store=f0d0b02e54813e28a58a04c19edb8553/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:19,501 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,505 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:19,505 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0d0b02e54813e28a58a04c19edb8553 columnFamilyName B 2024-11-18T03:15:19,505 DEBUG [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:19,506 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(327): Store=f0d0b02e54813e28a58a04c19edb8553/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:19,506 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,507 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:19,507 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0d0b02e54813e28a58a04c19edb8553 columnFamilyName C 2024-11-18T03:15:19,507 DEBUG [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:19,508 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(327): Store=f0d0b02e54813e28a58a04c19edb8553/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:19,508 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,508 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,509 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,509 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,510 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,510 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,511 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:15:19,513 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,517 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:15:19,518 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened f0d0b02e54813e28a58a04c19edb8553; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66494521, jitterRate=-0.00915442407131195}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:15:19,518 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:19,519 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for f0d0b02e54813e28a58a04c19edb8553: Running coprocessor pre-open hook at 1731899719495Writing region info on filesystem at 1731899719495Initializing all the Stores at 1731899719496 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899719496Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899719497 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899719497Cleaning up temporary data from old regions at 1731899719510 (+13 ms)Running coprocessor post-open hooks at 1731899719518 (+8 ms)Region opened successfully at 1731899719519 (+1 ms) 2024-11-18T03:15:19,520 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., pid=34, masterSystemTime=1731899719491 2024-11-18T03:15:19,523 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,523 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:19,523 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=OPEN, openSeqNum=2, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:19,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:15:19,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-18T03:15:19,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 in 189 msec 2024-11-18T03:15:19,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-18T03:15:19,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, ASSIGN in 348 msec 2024-11-18T03:15:19,534 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:15:19,534 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899719534"}]},"ts":"1731899719534"} 2024-11-18T03:15:19,536 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T03:15:19,537 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:15:19,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-18T03:15:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T03:15:20,502 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:20,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b4e2bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:20,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:20,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:20,505 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76669721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:20,507 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:20,509 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:20,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:20,514 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:15:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fd862c3a04f9,45051,1731899683325 2024-11-18T03:15:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@22d64e14 2024-11-18T03:15:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:15:20,517 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:15:20,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:15:20,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:15:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:20,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741962_1138 (size=1097) 2024-11-18T03:15:20,943 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T03:15:20,943 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T03:15:20,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:15:20,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, REOPEN/MOVE}] 2024-11-18T03:15:20,955 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, REOPEN/MOVE 2024-11-18T03:15:20,956 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:20,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, REOPEN/MOVE because future has completed 2024-11-18T03:15:20,958 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:15:20,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:15:21,110 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,111 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:15:21,111 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing f0d0b02e54813e28a58a04c19edb8553, disabling compactions & flushes 2024-11-18T03:15:21,111 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,111 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,111 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. after waiting 0 ms 2024-11-18T03:15:21,111 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,115 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T03:15:21,115 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,115 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for f0d0b02e54813e28a58a04c19edb8553: Waiting for close lock at 1731899721111Running coprocessor pre-close hooks at 1731899721111Disabling compacts and flushes for region at 1731899721111Disabling writes for close at 1731899721111Writing region close event to WAL at 1731899721111Running coprocessor post-close hooks at 1731899721115 (+4 ms)Closed at 1731899721115 2024-11-18T03:15:21,115 WARN [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: f0d0b02e54813e28a58a04c19edb8553 to self. 2024-11-18T03:15:21,118 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=CLOSED 2024-11-18T03:15:21,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:15:21,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-18T03:15:21,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 in 163 msec 2024-11-18T03:15:21,123 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, REOPEN/MOVE; state=CLOSED, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=true 2024-11-18T03:15:21,274 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, REOPEN/MOVE because future has completed 2024-11-18T03:15:21,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:15:21,431 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,431 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:15:21,432 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,432 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:15:21,432 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,432 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,435 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,436 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:21,442 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0d0b02e54813e28a58a04c19edb8553 columnFamilyName A 2024-11-18T03:15:21,445 DEBUG [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:21,445 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(327): Store=f0d0b02e54813e28a58a04c19edb8553/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:21,446 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,448 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:21,448 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0d0b02e54813e28a58a04c19edb8553 columnFamilyName B 2024-11-18T03:15:21,449 DEBUG [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:21,449 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(327): Store=f0d0b02e54813e28a58a04c19edb8553/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:21,449 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,450 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:21,450 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0d0b02e54813e28a58a04c19edb8553 columnFamilyName C 2024-11-18T03:15:21,451 DEBUG [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:21,451 INFO [StoreOpener-f0d0b02e54813e28a58a04c19edb8553-1 {}] regionserver.HStore(327): Store=f0d0b02e54813e28a58a04c19edb8553/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:21,451 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,451 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,452 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,453 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,454 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,454 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,454 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:15:21,455 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,456 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f0d0b02e54813e28a58a04c19edb8553; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61847254, jitterRate=-0.07840409874916077}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:15:21,456 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,458 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f0d0b02e54813e28a58a04c19edb8553: Running coprocessor pre-open hook at 1731899721432Writing region info on filesystem at 1731899721432Initializing all the Stores at 1731899721433 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899721433Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899721433Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899721434 (+1 ms)Cleaning up temporary data from old regions at 1731899721454 (+20 ms)Running coprocessor post-open hooks at 1731899721456 (+2 ms)Region opened successfully at 1731899721458 (+2 ms) 2024-11-18T03:15:21,459 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., pid=39, masterSystemTime=1731899721428 2024-11-18T03:15:21,461 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,462 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,462 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=OPEN, openSeqNum=5, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:15:21,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-18T03:15:21,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 in 193 msec 2024-11-18T03:15:21,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-18T03:15:21,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, REOPEN/MOVE in 517 msec 2024-11-18T03:15:21,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-18T03:15:21,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 540 msec 2024-11-18T03:15:21,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 984 msec 2024-11-18T03:15:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-18T03:15:21,517 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:21,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37870970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15cb3e3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,523 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61643f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454c5b1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,526 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9778420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,528 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383b693c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72aba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,534 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cefe2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,537 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5c2e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbd945b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,540 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4755d4ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,542 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477326e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d26dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,545 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a680526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,547 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2702c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182dc420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,554 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@172b2725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e396ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4500591c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eba169d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,567 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c0b19c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e853ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,569 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2baaac90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,571 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0f0da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6230e7af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:21,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:21,573 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a4a233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:21,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2edd6c6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:21,581 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,582 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,600 DEBUG [Thread-692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,600 DEBUG [Thread-701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:15:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:21,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:21,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,609 DEBUG [Thread-690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,626 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:21,630 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-18T03:15:21,632 DEBUG [Thread-699 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T03:15:21,632 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,632 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:21,632 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:21,633 DEBUG [Thread-695 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,633 DEBUG [Thread-695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731899721623, max timestamp=1731899721628Num uniques 859; 2024-11-18T03:15:21,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:15:21,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,637 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:21,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:21,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:15:21,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899721623, max timestamp=1731899721628Num uniques 859; 2024-11-18T03:15:21,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:15:21,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899721623, max timestamp=1731899721628Num uniques 859; 2024-11-18T03:15:21,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:15:21,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e01c2c231d28457b94c3b57d5c4f4d8c_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899721601/Put/seqid=0 2024-11-18T03:15:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899781684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899781690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899781693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899781695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899781695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741963_1139 (size=12154) 2024-11-18T03:15:21,720 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:21,730 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e01c2c231d28457b94c3b57d5c4f4d8c_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e01c2c231d28457b94c3b57d5c4f4d8c_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:21,732 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/b41febcd8e7a4a58b4524263f69ea70b, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:21,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/b41febcd8e7a4a58b4524263f69ea70b is 175, key is test_row_0/A:col10/1731899721601/Put/seqid=0 2024-11-18T03:15:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T03:15:21,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741964_1140 (size=30955) 2024-11-18T03:15:21,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/b41febcd8e7a4a58b4524263f69ea70b 2024-11-18T03:15:21,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T03:15:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:21,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899781805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899781811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899781814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899781811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899781821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:21,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/0192dd633c2e47c2a9e9dbad8219e81f is 50, key is test_row_0/B:col10/1731899721601/Put/seqid=0 2024-11-18T03:15:21,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741965_1141 (size=12001) 2024-11-18T03:15:21,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/0192dd633c2e47c2a9e9dbad8219e81f 2024-11-18T03:15:21,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T03:15:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:21,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T03:15:21,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/898ca1615a374e249c7b77f4dbef5c9c is 50, key is test_row_0/C:col10/1731899721601/Put/seqid=0 2024-11-18T03:15:21,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741966_1142 (size=12001) 2024-11-18T03:15:21,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/898ca1615a374e249c7b77f4dbef5c9c 2024-11-18T03:15:22,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/b41febcd8e7a4a58b4524263f69ea70b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b41febcd8e7a4a58b4524263f69ea70b 2024-11-18T03:15:22,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b41febcd8e7a4a58b4524263f69ea70b, entries=150, sequenceid=14, filesize=30.2 K 2024-11-18T03:15:22,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/0192dd633c2e47c2a9e9dbad8219e81f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/0192dd633c2e47c2a9e9dbad8219e81f 2024-11-18T03:15:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899782021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899782031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899782032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/0192dd633c2e47c2a9e9dbad8219e81f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T03:15:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899782032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899782032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/898ca1615a374e249c7b77f4dbef5c9c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/898ca1615a374e249c7b77f4dbef5c9c 2024-11-18T03:15:22,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/898ca1615a374e249c7b77f4dbef5c9c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T03:15:22,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f0d0b02e54813e28a58a04c19edb8553 in 444ms, sequenceid=14, compaction requested=false 2024-11-18T03:15:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:22,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T03:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:22,103 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-18T03:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:22,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118445b476e16e944b9bd7e1204f1b5c1af_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899721692/Put/seqid=0 2024-11-18T03:15:22,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741967_1143 (size=12154) 2024-11-18T03:15:22,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,195 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118445b476e16e944b9bd7e1204f1b5c1af_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118445b476e16e944b9bd7e1204f1b5c1af_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:22,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/445ecee55d124f5397bc51c1445d0477, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/445ecee55d124f5397bc51c1445d0477 is 175, key is test_row_0/A:col10/1731899721692/Put/seqid=0 2024-11-18T03:15:22,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741968_1144 (size=30955) 2024-11-18T03:15:22,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/445ecee55d124f5397bc51c1445d0477 2024-11-18T03:15:22,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/c91029d2754f45dcb633fc37906425b9 is 50, key is test_row_0/B:col10/1731899721692/Put/seqid=0 2024-11-18T03:15:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T03:15:22,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741969_1145 (size=12001) 2024-11-18T03:15:22,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/c91029d2754f45dcb633fc37906425b9 2024-11-18T03:15:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6a82f8e4c69542e283826aadc208b089 is 50, key is test_row_0/C:col10/1731899721692/Put/seqid=0 2024-11-18T03:15:22,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741970_1146 (size=12001) 2024-11-18T03:15:22,317 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6a82f8e4c69542e283826aadc208b089 2024-11-18T03:15:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/445ecee55d124f5397bc51c1445d0477 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/445ecee55d124f5397bc51c1445d0477 2024-11-18T03:15:22,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/445ecee55d124f5397bc51c1445d0477, entries=150, sequenceid=50, filesize=30.2 K 2024-11-18T03:15:22,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/c91029d2754f45dcb633fc37906425b9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/c91029d2754f45dcb633fc37906425b9 2024-11-18T03:15:22,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,353 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/c91029d2754f45dcb633fc37906425b9, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T03:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6a82f8e4c69542e283826aadc208b089 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6a82f8e4c69542e283826aadc208b089 2024-11-18T03:15:22,362 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6a82f8e4c69542e283826aadc208b089, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T03:15:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,365 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=26.84 KB/27480 for f0d0b02e54813e28a58a04c19edb8553 in 262ms, sequenceid=50, compaction requested=false 2024-11-18T03:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-18T03:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-18T03:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-18T03:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 731 msec 2024-11-18T03:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 743 msec 2024-11-18T03:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ff14fa6549a54d268d03af05bd3166d9_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899722388/Put/seqid=0 2024-11-18T03:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741971_1147 (size=21918) 2024-11-18T03:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,456 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ff14fa6549a54d268d03af05bd3166d9_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ff14fa6549a54d268d03af05bd3166d9_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/014ede28530c4a9fbb8547d46b3e1e09, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:22,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/014ede28530c4a9fbb8547d46b3e1e09 is 175, key is test_row_0/A:col10/1731899722388/Put/seqid=0 2024-11-18T03:15:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741972_1148 (size=65323) 2024-11-18T03:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899782508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731899722393, max timestamp=1731899722516Num uniques 859; 2024-11-18T03:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T03:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899782510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731899722393, max timestamp=1731899722516Num uniques 859; 2024-11-18T03:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899782519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T03:15:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731899722393, max timestamp=1731899722516Num uniques 859; 2024-11-18T03:15:22,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T03:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:22,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899782536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899782541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899782631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899782631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899782632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899782662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899782668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T03:15:22,773 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:22,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-18T03:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T03:15:22,778 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:22,779 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:22,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:22,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899782841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899782842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899782842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899782871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T03:15:22,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899782881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:22,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/014ede28530c4a9fbb8547d46b3e1e09 2024-11-18T03:15:22,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/62ac982ae53f440bbe85134770c58075 is 50, key is test_row_0/B:col10/1731899722388/Put/seqid=0 2024-11-18T03:15:22,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T03:15:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:22,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:22,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741973_1149 (size=12001) 2024-11-18T03:15:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T03:15:23,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T03:15:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899783151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899783151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899783151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899783180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899783191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T03:15:23,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:23,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,325 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:15:23,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/62ac982ae53f440bbe85134770c58075 2024-11-18T03:15:23,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6c184d724510441d847b891ba3ef53ad is 50, key is test_row_0/C:col10/1731899722388/Put/seqid=0 2024-11-18T03:15:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T03:15:23,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741974_1150 (size=12001) 2024-11-18T03:15:23,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6c184d724510441d847b891ba3ef53ad 2024-11-18T03:15:23,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T03:15:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:23,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/014ede28530c4a9fbb8547d46b3e1e09 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/014ede28530c4a9fbb8547d46b3e1e09 2024-11-18T03:15:23,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/014ede28530c4a9fbb8547d46b3e1e09, entries=350, sequenceid=60, filesize=63.8 K 2024-11-18T03:15:23,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/62ac982ae53f440bbe85134770c58075 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/62ac982ae53f440bbe85134770c58075 2024-11-18T03:15:23,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/62ac982ae53f440bbe85134770c58075, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T03:15:23,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6c184d724510441d847b891ba3ef53ad as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6c184d724510441d847b891ba3ef53ad 2024-11-18T03:15:23,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6c184d724510441d847b891ba3ef53ad, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T03:15:23,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for f0d0b02e54813e28a58a04c19edb8553 in 1051ms, sequenceid=60, compaction requested=true 2024-11-18T03:15:23,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:23,443 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:23,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:23,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:23,444 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:23,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:23,445 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:23,445 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:23,445 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,445 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b41febcd8e7a4a58b4524263f69ea70b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/445ecee55d124f5397bc51c1445d0477, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/014ede28530c4a9fbb8547d46b3e1e09] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=124.3 K 2024-11-18T03:15:23,445 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,445 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b41febcd8e7a4a58b4524263f69ea70b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/445ecee55d124f5397bc51c1445d0477, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/014ede28530c4a9fbb8547d46b3e1e09] 2024-11-18T03:15:23,446 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:23,446 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:23,446 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,446 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/0192dd633c2e47c2a9e9dbad8219e81f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/c91029d2754f45dcb633fc37906425b9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/62ac982ae53f440bbe85134770c58075] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.2 K 2024-11-18T03:15:23,446 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting b41febcd8e7a4a58b4524263f69ea70b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731899721594 2024-11-18T03:15:23,447 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0192dd633c2e47c2a9e9dbad8219e81f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731899721594 2024-11-18T03:15:23,447 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c91029d2754f45dcb633fc37906425b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731899721689 2024-11-18T03:15:23,447 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 445ecee55d124f5397bc51c1445d0477, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731899721689 2024-11-18T03:15:23,447 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ac982ae53f440bbe85134770c58075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731899722352 2024-11-18T03:15:23,447 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014ede28530c4a9fbb8547d46b3e1e09, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731899722350 2024-11-18T03:15:23,461 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:23,464 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:23,465 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/1dd1fd9ff87841598e0d65a413e2f7ce is 50, key is test_row_0/B:col10/1731899722388/Put/seqid=0 2024-11-18T03:15:23,465 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411188f691c8da8204883a3c994c8828526e7_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:23,473 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411188f691c8da8204883a3c994c8828526e7_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:23,473 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188f691c8da8204883a3c994c8828526e7_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:23,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741975_1151 (size=12104) 2024-11-18T03:15:23,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741976_1152 (size=4469) 2024-11-18T03:15:23,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T03:15:23,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,569 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-18T03:15:23,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:23,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:23,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:23,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:23,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:23,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:23,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b3816c9b080344c09c594281214389d7_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899722524/Put/seqid=0 2024-11-18T03:15:23,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741977_1153 (size=12154) 2024-11-18T03:15:23,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:23,660 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b3816c9b080344c09c594281214389d7_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b3816c9b080344c09c594281214389d7_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2c2b238fedca4c5984496705d4bfc22d, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2c2b238fedca4c5984496705d4bfc22d is 175, key is test_row_0/A:col10/1731899722524/Put/seqid=0 2024-11-18T03:15:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:23,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:23,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741978_1154 (size=30955) 2024-11-18T03:15:23,706 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2c2b238fedca4c5984496705d4bfc22d 2024-11-18T03:15:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899783691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899783693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899783701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899783702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/da73b16e35924879a3c244374d7c38f9 is 50, key is test_row_0/B:col10/1731899722524/Put/seqid=0 2024-11-18T03:15:23,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899783706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741979_1155 (size=12001) 2024-11-18T03:15:23,751 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/da73b16e35924879a3c244374d7c38f9 2024-11-18T03:15:23,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/1659a3e15df045a6b092736dcaedd7f5 is 50, key is test_row_0/C:col10/1731899722524/Put/seqid=0 2024-11-18T03:15:23,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741980_1156 (size=12001) 2024-11-18T03:15:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899783811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899783824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899783831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:23,907 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#132 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:23,908 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/4229e9cdef4c430b9bc66192d467b504 is 175, key is test_row_0/A:col10/1731899722388/Put/seqid=0 2024-11-18T03:15:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T03:15:23,917 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/1dd1fd9ff87841598e0d65a413e2f7ce as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1dd1fd9ff87841598e0d65a413e2f7ce 2024-11-18T03:15:23,927 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into 1dd1fd9ff87841598e0d65a413e2f7ce(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:23,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:23,928 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899723443; duration=0sec 2024-11-18T03:15:23,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:23,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:23,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:23,930 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:23,930 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:23,930 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:23,931 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/898ca1615a374e249c7b77f4dbef5c9c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6a82f8e4c69542e283826aadc208b089, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6c184d724510441d847b891ba3ef53ad] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.2 K 2024-11-18T03:15:23,932 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 898ca1615a374e249c7b77f4dbef5c9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731899721594 2024-11-18T03:15:23,932 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a82f8e4c69542e283826aadc208b089, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731899721689 2024-11-18T03:15:23,935 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c184d724510441d847b891ba3ef53ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731899722352 2024-11-18T03:15:23,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741981_1157 (size=31058) 2024-11-18T03:15:23,954 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/4229e9cdef4c430b9bc66192d467b504 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/4229e9cdef4c430b9bc66192d467b504 2024-11-18T03:15:23,965 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into 4229e9cdef4c430b9bc66192d467b504(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:23,965 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:23,965 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899723443; duration=0sec 2024-11-18T03:15:23,965 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:23,965 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:23,971 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:23,976 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/020e71f905534dbb8f348845845e9767 is 50, key is test_row_0/C:col10/1731899722388/Put/seqid=0 2024-11-18T03:15:24,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741982_1158 (size=12104) 2024-11-18T03:15:24,027 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/020e71f905534dbb8f348845845e9767 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/020e71f905534dbb8f348845845e9767 2024-11-18T03:15:24,036 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into 020e71f905534dbb8f348845845e9767(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:24,037 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:24,037 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899723445; duration=0sec 2024-11-18T03:15:24,037 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:24,037 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899784032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899784041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899784068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,191 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/1659a3e15df045a6b092736dcaedd7f5 2024-11-18T03:15:24,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2c2b238fedca4c5984496705d4bfc22d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2c2b238fedca4c5984496705d4bfc22d 2024-11-18T03:15:24,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2c2b238fedca4c5984496705d4bfc22d, entries=150, sequenceid=86, filesize=30.2 K 2024-11-18T03:15:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/da73b16e35924879a3c244374d7c38f9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/da73b16e35924879a3c244374d7c38f9 2024-11-18T03:15:24,258 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/da73b16e35924879a3c244374d7c38f9, entries=150, sequenceid=86, filesize=11.7 K 2024-11-18T03:15:24,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/1659a3e15df045a6b092736dcaedd7f5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/1659a3e15df045a6b092736dcaedd7f5 2024-11-18T03:15:24,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/1659a3e15df045a6b092736dcaedd7f5, entries=150, sequenceid=86, filesize=11.7 K 2024-11-18T03:15:24,266 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=53.67 KB/54960 for f0d0b02e54813e28a58a04c19edb8553 in 697ms, sequenceid=86, compaction requested=false 2024-11-18T03:15:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-18T03:15:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-18T03:15:24,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-18T03:15:24,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4890 sec 2024-11-18T03:15:24,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4970 sec 2024-11-18T03:15:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:24,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T03:15:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:24,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cd282be6e9cb4d958bd399ba887507e3_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899723694/Put/seqid=0 2024-11-18T03:15:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741983_1159 (size=12154) 2024-11-18T03:15:24,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899784411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899784418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899784420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899784537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899784537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899784537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899784721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,723 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in f0d0b02e54813e28a58a04c19edb8553 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899784721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,727 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in f0d0b02e54813e28a58a04c19edb8553 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899784751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899784751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899784751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:24,792 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:24,803 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cd282be6e9cb4d958bd399ba887507e3_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cd282be6e9cb4d958bd399ba887507e3_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:24,805 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bada2586ccf54cb0a8182f2305e7d394, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:24,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bada2586ccf54cb0a8182f2305e7d394 is 175, key is test_row_0/A:col10/1731899723694/Put/seqid=0 2024-11-18T03:15:24,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741984_1160 (size=30955) 2024-11-18T03:15:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T03:15:24,921 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:24,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-18T03:15:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T03:15:24,925 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:24,926 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:24,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T03:15:25,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899785061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:25,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899785061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:25,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899785062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:25,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T03:15:25,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bada2586ccf54cb0a8182f2305e7d394 2024-11-18T03:15:25,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T03:15:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/f98af1634d79409cb4037f85c53b6003 is 50, key is test_row_0/B:col10/1731899723694/Put/seqid=0 2024-11-18T03:15:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T03:15:25,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741985_1161 (size=12001) 2024-11-18T03:15:25,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/f98af1634d79409cb4037f85c53b6003 2024-11-18T03:15:25,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/b9c43a6f17324419be83b68c49ab933f is 50, key is test_row_0/C:col10/1731899723694/Put/seqid=0 2024-11-18T03:15:25,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741986_1162 (size=12001) 2024-11-18T03:15:25,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T03:15:25,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T03:15:25,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:25,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T03:15:25,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899785571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:25,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899785571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899785572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:25,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/b9c43a6f17324419be83b68c49ab933f 2024-11-18T03:15:25,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T03:15:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:25,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bada2586ccf54cb0a8182f2305e7d394 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bada2586ccf54cb0a8182f2305e7d394 2024-11-18T03:15:25,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bada2586ccf54cb0a8182f2305e7d394, entries=150, sequenceid=102, filesize=30.2 K 2024-11-18T03:15:25,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/f98af1634d79409cb4037f85c53b6003 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f98af1634d79409cb4037f85c53b6003 2024-11-18T03:15:25,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f98af1634d79409cb4037f85c53b6003, entries=150, sequenceid=102, filesize=11.7 K 2024-11-18T03:15:25,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/b9c43a6f17324419be83b68c49ab933f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/b9c43a6f17324419be83b68c49ab933f 2024-11-18T03:15:25,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/b9c43a6f17324419be83b68c49ab933f, entries=150, sequenceid=102, filesize=11.7 K 2024-11-18T03:15:25,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f0d0b02e54813e28a58a04c19edb8553 in 1364ms, sequenceid=102, compaction requested=true 2024-11-18T03:15:25,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:25,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:25,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:25,724 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:25,724 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:25,726 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:25,726 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:25,726 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,726 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/4229e9cdef4c430b9bc66192d467b504, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2c2b238fedca4c5984496705d4bfc22d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bada2586ccf54cb0a8182f2305e7d394] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=90.8 K 2024-11-18T03:15:25,726 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,726 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/4229e9cdef4c430b9bc66192d467b504, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2c2b238fedca4c5984496705d4bfc22d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bada2586ccf54cb0a8182f2305e7d394] 2024-11-18T03:15:25,727 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4229e9cdef4c430b9bc66192d467b504, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731899722352 2024-11-18T03:15:25,727 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:25,727 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:25,727 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c2b238fedca4c5984496705d4bfc22d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731899722524 2024-11-18T03:15:25,727 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,727 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1dd1fd9ff87841598e0d65a413e2f7ce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/da73b16e35924879a3c244374d7c38f9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f98af1634d79409cb4037f85c53b6003] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.3 K 2024-11-18T03:15:25,728 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting bada2586ccf54cb0a8182f2305e7d394, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899723694 2024-11-18T03:15:25,728 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd1fd9ff87841598e0d65a413e2f7ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731899722352 2024-11-18T03:15:25,729 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting da73b16e35924879a3c244374d7c38f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731899722524 2024-11-18T03:15:25,729 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f98af1634d79409cb4037f85c53b6003, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899723694 2024-11-18T03:15:25,746 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:25,746 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:25,747 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ca0bd1ead9ca4f39b7f2aade62cf4cc8 is 50, key is test_row_0/B:col10/1731899723694/Put/seqid=0 2024-11-18T03:15:25,757 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411184f0c402a8c7947ef8d56134d2c08e91d_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:25,760 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411184f0c402a8c7947ef8d56134d2c08e91d_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:25,760 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184f0c402a8c7947ef8d56134d2c08e91d_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:25,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741987_1163 (size=12207) 2024-11-18T03:15:25,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741988_1164 (size=4469) 2024-11-18T03:15:25,825 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ca0bd1ead9ca4f39b7f2aade62cf4cc8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ca0bd1ead9ca4f39b7f2aade62cf4cc8 2024-11-18T03:15:25,827 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#142 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:25,827 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/205667e0628c46d7af8c10a6b069cab0 is 175, key is test_row_0/A:col10/1731899723694/Put/seqid=0 2024-11-18T03:15:25,847 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into ca0bd1ead9ca4f39b7f2aade62cf4cc8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:25,847 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:25,847 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899725724; duration=0sec 2024-11-18T03:15:25,847 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:25,847 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:25,847 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:25,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T03:15:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,851 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:25,851 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T03:15:25,851 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:25,851 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:25,852 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/020e71f905534dbb8f348845845e9767, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/1659a3e15df045a6b092736dcaedd7f5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/b9c43a6f17324419be83b68c49ab933f] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.3 K 2024-11-18T03:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:25,853 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 020e71f905534dbb8f348845845e9767, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731899722352 2024-11-18T03:15:25,854 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1659a3e15df045a6b092736dcaedd7f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731899722524 2024-11-18T03:15:25,855 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b9c43a6f17324419be83b68c49ab933f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899723694 2024-11-18T03:15:25,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741989_1165 (size=31161) 2024-11-18T03:15:25,870 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/205667e0628c46d7af8c10a6b069cab0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/205667e0628c46d7af8c10a6b069cab0 2024-11-18T03:15:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111801fd7ca647b14fa0b86c341914901206_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899724415/Put/seqid=0 2024-11-18T03:15:25,884 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into 205667e0628c46d7af8c10a6b069cab0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:25,884 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:25,884 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899725724; duration=0sec 2024-11-18T03:15:25,884 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:25,885 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:25,886 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:25,887 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/96f9e4d388e340e3a8591f993153f586 is 50, key is test_row_0/C:col10/1731899723694/Put/seqid=0 2024-11-18T03:15:25,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741990_1166 (size=12154) 2024-11-18T03:15:25,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741991_1167 (size=12207) 2024-11-18T03:15:25,923 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/96f9e4d388e340e3a8591f993153f586 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/96f9e4d388e340e3a8591f993153f586 2024-11-18T03:15:25,932 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into 96f9e4d388e340e3a8591f993153f586(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:25,932 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:25,932 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899725725; duration=0sec 2024-11-18T03:15:25,932 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:25,932 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T03:15:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111801fd7ca647b14fa0b86c341914901206_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111801fd7ca647b14fa0b86c341914901206_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:26,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/f836a55e83db48a584387c18f9561c2f, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/f836a55e83db48a584387c18f9561c2f is 175, key is test_row_0/A:col10/1731899724415/Put/seqid=0 2024-11-18T03:15:26,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741992_1168 (size=30955) 2024-11-18T03:15:26,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/f836a55e83db48a584387c18f9561c2f 2024-11-18T03:15:26,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/8d6e5ed3225b417ba305fb101330781a is 50, key is test_row_0/B:col10/1731899724415/Put/seqid=0 2024-11-18T03:15:26,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741993_1169 (size=12001) 2024-11-18T03:15:26,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/8d6e5ed3225b417ba305fb101330781a 2024-11-18T03:15:26,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/d84edc32e5fc4854a8a4396812d2339d is 50, key is test_row_0/C:col10/1731899724415/Put/seqid=0 2024-11-18T03:15:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741994_1170 (size=12001) 2024-11-18T03:15:26,413 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/d84edc32e5fc4854a8a4396812d2339d 2024-11-18T03:15:26,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/f836a55e83db48a584387c18f9561c2f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/f836a55e83db48a584387c18f9561c2f 2024-11-18T03:15:26,421 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/f836a55e83db48a584387c18f9561c2f, entries=150, sequenceid=122, filesize=30.2 K 2024-11-18T03:15:26,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/8d6e5ed3225b417ba305fb101330781a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/8d6e5ed3225b417ba305fb101330781a 2024-11-18T03:15:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,426 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/8d6e5ed3225b417ba305fb101330781a, entries=150, sequenceid=122, filesize=11.7 K 2024-11-18T03:15:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/d84edc32e5fc4854a8a4396812d2339d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/d84edc32e5fc4854a8a4396812d2339d 2024-11-18T03:15:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,450 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/d84edc32e5fc4854a8a4396812d2339d, entries=150, sequenceid=122, filesize=11.7 K 2024-11-18T03:15:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,451 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for f0d0b02e54813e28a58a04c19edb8553 in 600ms, sequenceid=122, compaction requested=false 2024-11-18T03:15:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-18T03:15:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-18T03:15:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-18T03:15:26,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5360 sec 2024-11-18T03:15:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5480 sec 2024-11-18T03:15:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:26,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:15:26,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:26,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:26,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:26,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:26,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:26,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ee065d1795a5408e9e5ba449f8b2a8d3_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899726616/Put/seqid=0 2024-11-18T03:15:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741995_1171 (size=22218) 2024-11-18T03:15:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731899726622, max timestamp=1731899726681Num uniques 859; 2024-11-18T03:15:26,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731899726622, max timestamp=1731899726681Num uniques 859; 2024-11-18T03:15:26,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T03:15:26,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T03:15:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731899726622, max timestamp=1731899726681Num uniques 859; 2024-11-18T03:15:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T03:15:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899786696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899786700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899786704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:26,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899786741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,749 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in f0d0b02e54813e28a58a04c19edb8553 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899786751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,752 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in f0d0b02e54813e28a58a04c19edb8553 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:26,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899786811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899786811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:26,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899786821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899787021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899787021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899787031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:27,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ee065d1795a5408e9e5ba449f8b2a8d3_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee065d1795a5408e9e5ba449f8b2a8d3_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:27,072 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/65c18ec1a74846c0948edf0fe8b5f773, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:27,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/65c18ec1a74846c0948edf0fe8b5f773 is 175, key is test_row_0/A:col10/1731899726616/Put/seqid=0 2024-11-18T03:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T03:15:27,082 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:27,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-18T03:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T03:15:27,086 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:27,089 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:27,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:27,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741996_1172 (size=65623) 2024-11-18T03:15:27,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/65c18ec1a74846c0948edf0fe8b5f773 2024-11-18T03:15:27,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/d326480446fd478ab4c1766e33a01456 is 50, key is test_row_0/B:col10/1731899726616/Put/seqid=0 2024-11-18T03:15:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T03:15:27,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741997_1173 (size=12151) 2024-11-18T03:15:27,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T03:15:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899787331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899787331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899787341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T03:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T03:15:27,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T03:15:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/d326480446fd478ab4c1766e33a01456 2024-11-18T03:15:27,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/95dbb5609dd3401386146df9634cf0f0 is 50, key is test_row_0/C:col10/1731899726616/Put/seqid=0 2024-11-18T03:15:27,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741998_1174 (size=12151) 2024-11-18T03:15:27,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T03:15:27,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T03:15:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899787851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T03:15:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899787851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899787851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:28,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T03:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:28,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:28,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/95dbb5609dd3401386146df9634cf0f0 2024-11-18T03:15:28,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/65c18ec1a74846c0948edf0fe8b5f773 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/65c18ec1a74846c0948edf0fe8b5f773 2024-11-18T03:15:28,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/65c18ec1a74846c0948edf0fe8b5f773, entries=350, sequenceid=134, filesize=64.1 K 2024-11-18T03:15:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/d326480446fd478ab4c1766e33a01456 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d326480446fd478ab4c1766e33a01456 2024-11-18T03:15:28,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d326480446fd478ab4c1766e33a01456, entries=150, sequenceid=134, filesize=11.9 K 2024-11-18T03:15:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/95dbb5609dd3401386146df9634cf0f0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/95dbb5609dd3401386146df9634cf0f0 2024-11-18T03:15:28,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/95dbb5609dd3401386146df9634cf0f0, entries=150, sequenceid=134, filesize=11.9 K 2024-11-18T03:15:28,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for f0d0b02e54813e28a58a04c19edb8553 in 1419ms, sequenceid=134, compaction requested=true 2024-11-18T03:15:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:28,038 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:28,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:28,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:28,039 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:28,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:28,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:28,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:28,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:28,039 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:28,039 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:28,040 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:28,040 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:28,040 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:28,040 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ca0bd1ead9ca4f39b7f2aade62cf4cc8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/8d6e5ed3225b417ba305fb101330781a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d326480446fd478ab4c1766e33a01456] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.5 K 2024-11-18T03:15:28,040 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:28,040 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/205667e0628c46d7af8c10a6b069cab0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/f836a55e83db48a584387c18f9561c2f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/65c18ec1a74846c0948edf0fe8b5f773] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=124.7 K 2024-11-18T03:15:28,040 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:28,040 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/205667e0628c46d7af8c10a6b069cab0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/f836a55e83db48a584387c18f9561c2f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/65c18ec1a74846c0948edf0fe8b5f773] 2024-11-18T03:15:28,041 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0bd1ead9ca4f39b7f2aade62cf4cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899723694 2024-11-18T03:15:28,041 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 205667e0628c46d7af8c10a6b069cab0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899723694 2024-11-18T03:15:28,041 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6e5ed3225b417ba305fb101330781a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731899724407 2024-11-18T03:15:28,041 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f836a55e83db48a584387c18f9561c2f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731899724407 2024-11-18T03:15:28,042 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d326480446fd478ab4c1766e33a01456, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731899726614 2024-11-18T03:15:28,042 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65c18ec1a74846c0948edf0fe8b5f773, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731899726603 2024-11-18T03:15:28,051 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:28,051 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:28,052 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/1b36751c9f33498781bb0c4d706b0003 is 50, key is test_row_0/B:col10/1731899726616/Put/seqid=0 2024-11-18T03:15:28,055 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118c0be066dd8c9472ea3b65cb776c072f1_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:28,057 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118c0be066dd8c9472ea3b65cb776c072f1_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:28,058 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c0be066dd8c9472ea3b65cb776c072f1_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:28,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741999_1175 (size=12459) 2024-11-18T03:15:28,070 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/1b36751c9f33498781bb0c4d706b0003 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1b36751c9f33498781bb0c4d706b0003 2024-11-18T03:15:28,076 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into 1b36751c9f33498781bb0c4d706b0003(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:28,076 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:28,076 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899728039; duration=0sec 2024-11-18T03:15:28,076 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:28,076 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:28,076 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:28,077 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:28,078 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:28,078 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:28,078 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/96f9e4d388e340e3a8591f993153f586, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/d84edc32e5fc4854a8a4396812d2339d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/95dbb5609dd3401386146df9634cf0f0] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.5 K 2024-11-18T03:15:28,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742000_1176 (size=4469) 2024-11-18T03:15:28,080 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f9e4d388e340e3a8591f993153f586, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899723694 2024-11-18T03:15:28,080 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d84edc32e5fc4854a8a4396812d2339d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731899724407 2024-11-18T03:15:28,081 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#151 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:28,081 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/9ad6960c37ec4cbb87ef19718fa07580 is 175, key is test_row_0/A:col10/1731899726616/Put/seqid=0 2024-11-18T03:15:28,083 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 95dbb5609dd3401386146df9634cf0f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731899726614 2024-11-18T03:15:28,095 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:28,095 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/cac375f78ec34236b4010a7020ba94ad is 50, key is test_row_0/C:col10/1731899726616/Put/seqid=0 2024-11-18T03:15:28,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742001_1177 (size=31413) 2024-11-18T03:15:28,106 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/9ad6960c37ec4cbb87ef19718fa07580 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9ad6960c37ec4cbb87ef19718fa07580 2024-11-18T03:15:28,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742002_1178 (size=12459) 2024-11-18T03:15:28,116 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into 9ad6960c37ec4cbb87ef19718fa07580(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:28,116 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:28,116 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899728038; duration=0sec 2024-11-18T03:15:28,116 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:28,116 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:28,127 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/cac375f78ec34236b4010a7020ba94ad as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/cac375f78ec34236b4010a7020ba94ad 2024-11-18T03:15:28,134 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into cac375f78ec34236b4010a7020ba94ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:28,134 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:28,134 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899728039; duration=0sec 2024-11-18T03:15:28,134 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:28,134 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:28,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T03:15:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:28,161 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-18T03:15:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118478c9ae7acb34b29961c14a7f63cc8bf_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899726699/Put/seqid=0 2024-11-18T03:15:28,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742003_1179 (size=12304) 2024-11-18T03:15:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:28,203 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118478c9ae7acb34b29961c14a7f63cc8bf_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118478c9ae7acb34b29961c14a7f63cc8bf_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2cb938be5e0e460d9a6e5f48287142df, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:28,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2cb938be5e0e460d9a6e5f48287142df is 175, key is test_row_0/A:col10/1731899726699/Put/seqid=0 2024-11-18T03:15:28,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742004_1180 (size=31105) 2024-11-18T03:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T03:15:28,623 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2cb938be5e0e460d9a6e5f48287142df 2024-11-18T03:15:28,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/88ce63ef3dd341839e3fe747b0ca469f is 50, key is test_row_0/B:col10/1731899726699/Put/seqid=0 2024-11-18T03:15:28,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742005_1181 (size=12151) 2024-11-18T03:15:28,640 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/88ce63ef3dd341839e3fe747b0ca469f 2024-11-18T03:15:28,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/13e7408b28b2401da41c9b8a4af56734 is 50, key is test_row_0/C:col10/1731899726699/Put/seqid=0 2024-11-18T03:15:28,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742006_1182 (size=12151) 2024-11-18T03:15:28,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:28,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899788881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899788882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899788882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:28,927 INFO [master/fd862c3a04f9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T03:15:28,927 INFO [master/fd862c3a04f9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T03:15:29,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899788993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899788994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899788996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/13e7408b28b2401da41c9b8a4af56734 2024-11-18T03:15:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/2cb938be5e0e460d9a6e5f48287142df as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2cb938be5e0e460d9a6e5f48287142df 2024-11-18T03:15:29,076 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2cb938be5e0e460d9a6e5f48287142df, entries=150, sequenceid=163, filesize=30.4 K 2024-11-18T03:15:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/88ce63ef3dd341839e3fe747b0ca469f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/88ce63ef3dd341839e3fe747b0ca469f 2024-11-18T03:15:29,088 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/88ce63ef3dd341839e3fe747b0ca469f, entries=150, sequenceid=163, filesize=11.9 K 2024-11-18T03:15:29,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/13e7408b28b2401da41c9b8a4af56734 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/13e7408b28b2401da41c9b8a4af56734 2024-11-18T03:15:29,104 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/13e7408b28b2401da41c9b8a4af56734, entries=150, sequenceid=163, filesize=11.9 K 2024-11-18T03:15:29,106 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=53.67 KB/54960 for f0d0b02e54813e28a58a04c19edb8553 in 945ms, sequenceid=163, compaction requested=false 2024-11-18T03:15:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-18T03:15:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-18T03:15:29,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-18T03:15:29,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-18T03:15:29,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0360 sec 2024-11-18T03:15:29,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T03:15:29,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:29,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:29,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:29,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:29,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:29,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T03:15:29,232 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:29,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-18T03:15:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T03:15:29,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:29,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:29,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:29,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186c3e2482ec6a45038edbe3e06691defa_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899728881/Put/seqid=0 2024-11-18T03:15:29,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742007_1183 (size=14794) 2024-11-18T03:15:29,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:29,291 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186c3e2482ec6a45038edbe3e06691defa_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186c3e2482ec6a45038edbe3e06691defa_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:29,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899789286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,296 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/97e0e60a447c403196307947674e3538, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:29,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/97e0e60a447c403196307947674e3538 is 175, key is test_row_0/A:col10/1731899728881/Put/seqid=0 2024-11-18T03:15:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899789294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899789293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742008_1184 (size=39749) 2024-11-18T03:15:29,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/97e0e60a447c403196307947674e3538 2024-11-18T03:15:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T03:15:29,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/98597b277d9b4f54a2517413099eed14 is 50, key is test_row_0/B:col10/1731899728881/Put/seqid=0 2024-11-18T03:15:29,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T03:15:29,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:29,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:29,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:29,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742009_1185 (size=12151) 2024-11-18T03:15:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:29,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/98597b277d9b4f54a2517413099eed14 2024-11-18T03:15:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899789401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899789401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899789411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/7153e1c0db094546a0115feed101826c is 50, key is test_row_0/C:col10/1731899728881/Put/seqid=0 2024-11-18T03:15:29,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742010_1186 (size=12151) 2024-11-18T03:15:29,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/7153e1c0db094546a0115feed101826c 2024-11-18T03:15:29,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/97e0e60a447c403196307947674e3538 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/97e0e60a447c403196307947674e3538 2024-11-18T03:15:29,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/97e0e60a447c403196307947674e3538, entries=200, sequenceid=175, filesize=38.8 K 2024-11-18T03:15:29,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/98597b277d9b4f54a2517413099eed14 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/98597b277d9b4f54a2517413099eed14 2024-11-18T03:15:29,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/98597b277d9b4f54a2517413099eed14, entries=150, sequenceid=175, filesize=11.9 K 2024-11-18T03:15:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/7153e1c0db094546a0115feed101826c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/7153e1c0db094546a0115feed101826c 2024-11-18T03:15:29,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/7153e1c0db094546a0115feed101826c, entries=150, sequenceid=175, filesize=11.9 K 2024-11-18T03:15:29,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f0d0b02e54813e28a58a04c19edb8553 in 316ms, sequenceid=175, compaction requested=true 2024-11-18T03:15:29,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:29,544 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:29,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:29,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:29,544 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:29,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:29,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:29,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:29,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:29,545 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:29,545 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:29,545 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,546 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:29,546 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:29,546 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,546 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9ad6960c37ec4cbb87ef19718fa07580, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2cb938be5e0e460d9a6e5f48287142df, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/97e0e60a447c403196307947674e3538] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=99.9 K 2024-11-18T03:15:29,546 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,546 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9ad6960c37ec4cbb87ef19718fa07580, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2cb938be5e0e460d9a6e5f48287142df, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/97e0e60a447c403196307947674e3538] 2024-11-18T03:15:29,546 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1b36751c9f33498781bb0c4d706b0003, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/88ce63ef3dd341839e3fe747b0ca469f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/98597b277d9b4f54a2517413099eed14] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.9 K 2024-11-18T03:15:29,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T03:15:29,547 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad6960c37ec4cbb87ef19718fa07580, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731899726614 2024-11-18T03:15:29,547 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b36751c9f33498781bb0c4d706b0003, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731899726614 2024-11-18T03:15:29,547 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb938be5e0e460d9a6e5f48287142df, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899726694 2024-11-18T03:15:29,548 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e0e60a447c403196307947674e3538, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731899728863 2024-11-18T03:15:29,548 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ce63ef3dd341839e3fe747b0ca469f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899726694 2024-11-18T03:15:29,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,549 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T03:15:29,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:29,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:29,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:29,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:29,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:29,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:29,551 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 98597b277d9b4f54a2517413099eed14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731899728881 2024-11-18T03:15:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T03:15:29,577 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:29,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118368f706011a24ce18cc80169214ea74f_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899729283/Put/seqid=0 2024-11-18T03:15:29,601 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:29,602 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/b079d753b7c64f91a262f3f2bb1f24ed is 50, key is test_row_0/B:col10/1731899728881/Put/seqid=0 2024-11-18T03:15:29,610 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118a88b0b7bc12a404dadd8fb87020552a5_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:29,612 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118a88b0b7bc12a404dadd8fb87020552a5_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:29,613 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a88b0b7bc12a404dadd8fb87020552a5_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:29,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:29,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742011_1187 (size=12304) 2024-11-18T03:15:29,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:29,654 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118368f706011a24ce18cc80169214ea74f_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118368f706011a24ce18cc80169214ea74f_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/81c260551c3d4671b1098c002f66087b, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/81c260551c3d4671b1098c002f66087b is 175, key is test_row_0/A:col10/1731899729283/Put/seqid=0 2024-11-18T03:15:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899789656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899789662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899789663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742013_1189 (size=4469) 2024-11-18T03:15:29,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742014_1190 (size=31105) 2024-11-18T03:15:29,692 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/81c260551c3d4671b1098c002f66087b 2024-11-18T03:15:29,693 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#159 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:29,694 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/b577a36dcd9b4f88967b288897756ad8 is 175, key is test_row_0/A:col10/1731899728881/Put/seqid=0 2024-11-18T03:15:29,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742012_1188 (size=12561) 2024-11-18T03:15:29,707 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/b079d753b7c64f91a262f3f2bb1f24ed as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b079d753b7c64f91a262f3f2bb1f24ed 2024-11-18T03:15:29,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742015_1191 (size=31515) 2024-11-18T03:15:29,716 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into b079d753b7c64f91a262f3f2bb1f24ed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:29,716 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:29,716 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899729544; duration=0sec 2024-11-18T03:15:29,716 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:29,716 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:29,716 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:29,717 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:29,717 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:29,717 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:29,717 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/cac375f78ec34236b4010a7020ba94ad, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/13e7408b28b2401da41c9b8a4af56734, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/7153e1c0db094546a0115feed101826c] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=35.9 K 2024-11-18T03:15:29,718 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting cac375f78ec34236b4010a7020ba94ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731899726614 2024-11-18T03:15:29,718 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 13e7408b28b2401da41c9b8a4af56734, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899726694 2024-11-18T03:15:29,719 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7153e1c0db094546a0115feed101826c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731899728881 2024-11-18T03:15:29,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/faaa9f5fdda0432283310e52dbaeed29 is 50, key is test_row_0/B:col10/1731899729283/Put/seqid=0 2024-11-18T03:15:29,739 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:29,739 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/e98a5074f7d8450c9ac13ba1dd34644d is 50, key is test_row_0/C:col10/1731899728881/Put/seqid=0 2024-11-18T03:15:29,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742016_1192 (size=12151) 2024-11-18T03:15:29,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742017_1193 (size=12561) 2024-11-18T03:15:29,769 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/e98a5074f7d8450c9ac13ba1dd34644d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/e98a5074f7d8450c9ac13ba1dd34644d 2024-11-18T03:15:29,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899789771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899789771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,778 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into e98a5074f7d8450c9ac13ba1dd34644d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:29,778 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:29,778 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899729545; duration=0sec 2024-11-18T03:15:29,778 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:29,778 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899789780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T03:15:29,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899789980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899789980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:29,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899789991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,129 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/b577a36dcd9b4f88967b288897756ad8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b577a36dcd9b4f88967b288897756ad8 2024-11-18T03:15:30,139 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into b577a36dcd9b4f88967b288897756ad8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:30,139 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:30,139 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899729544; duration=0sec 2024-11-18T03:15:30,139 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:30,139 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:30,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/faaa9f5fdda0432283310e52dbaeed29 2024-11-18T03:15:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/bb978f13aacc433ebc4752d2fa01f45d is 50, key is test_row_0/C:col10/1731899729283/Put/seqid=0 2024-11-18T03:15:30,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742018_1194 (size=12151) 2024-11-18T03:15:30,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899790291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899790291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899790301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T03:15:30,585 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/bb978f13aacc433ebc4752d2fa01f45d 2024-11-18T03:15:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/81c260551c3d4671b1098c002f66087b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/81c260551c3d4671b1098c002f66087b 2024-11-18T03:15:30,599 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/81c260551c3d4671b1098c002f66087b, entries=150, sequenceid=195, filesize=30.4 K 2024-11-18T03:15:30,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/faaa9f5fdda0432283310e52dbaeed29 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/faaa9f5fdda0432283310e52dbaeed29 2024-11-18T03:15:30,623 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/faaa9f5fdda0432283310e52dbaeed29, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T03:15:30,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/bb978f13aacc433ebc4752d2fa01f45d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bb978f13aacc433ebc4752d2fa01f45d 2024-11-18T03:15:30,635 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bb978f13aacc433ebc4752d2fa01f45d, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T03:15:30,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f0d0b02e54813e28a58a04c19edb8553 in 1093ms, sequenceid=195, compaction requested=false 2024-11-18T03:15:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:30,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-18T03:15:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-18T03:15:30,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-18T03:15:30,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-18T03:15:30,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4150 sec 2024-11-18T03:15:30,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T03:15:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:30,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:30,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:30,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:30,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:30,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:30,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:30,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e0af5119555b4403937c40ecf0910a4b_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899730789/Put/seqid=0 2024-11-18T03:15:30,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899790834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899790839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899790842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899790843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899790844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742019_1195 (size=14794) 2024-11-18T03:15:30,873 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:30,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e0af5119555b4403937c40ecf0910a4b_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e0af5119555b4403937c40ecf0910a4b_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:30,881 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0be546d76c38468180e5687b10fa4460, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:30,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0be546d76c38468180e5687b10fa4460 is 175, key is test_row_0/A:col10/1731899730789/Put/seqid=0 2024-11-18T03:15:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742020_1196 (size=39749) 2024-11-18T03:15:30,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0be546d76c38468180e5687b10fa4460 2024-11-18T03:15:30,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/414e10dfe41d460da8fb6dd2fa061ace is 50, key is test_row_0/B:col10/1731899730789/Put/seqid=0 2024-11-18T03:15:30,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899790951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899790961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899790971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899790971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899790971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:30,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742021_1197 (size=12151) 2024-11-18T03:15:30,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/414e10dfe41d460da8fb6dd2fa061ace 2024-11-18T03:15:31,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/18682de21e644df5934ef0e37c95e6a4 is 50, key is test_row_0/C:col10/1731899730789/Put/seqid=0 2024-11-18T03:15:31,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742022_1198 (size=12151) 2024-11-18T03:15:31,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/18682de21e644df5934ef0e37c95e6a4 2024-11-18T03:15:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0be546d76c38468180e5687b10fa4460 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0be546d76c38468180e5687b10fa4460 2024-11-18T03:15:31,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0be546d76c38468180e5687b10fa4460, entries=200, sequenceid=213, filesize=38.8 K 2024-11-18T03:15:31,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/414e10dfe41d460da8fb6dd2fa061ace as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/414e10dfe41d460da8fb6dd2fa061ace 2024-11-18T03:15:31,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/414e10dfe41d460da8fb6dd2fa061ace, entries=150, sequenceid=213, filesize=11.9 K 2024-11-18T03:15:31,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/18682de21e644df5934ef0e37c95e6a4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/18682de21e644df5934ef0e37c95e6a4 2024-11-18T03:15:31,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/18682de21e644df5934ef0e37c95e6a4, entries=150, sequenceid=213, filesize=11.9 K 2024-11-18T03:15:31,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f0d0b02e54813e28a58a04c19edb8553 in 284ms, sequenceid=213, compaction requested=true 2024-11-18T03:15:31,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:31,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:31,073 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:31,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:31,074 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:31,075 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:31,075 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:31,075 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,075 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b577a36dcd9b4f88967b288897756ad8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/81c260551c3d4671b1098c002f66087b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0be546d76c38468180e5687b10fa4460] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=100.0 K 2024-11-18T03:15:31,075 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,075 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b577a36dcd9b4f88967b288897756ad8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/81c260551c3d4671b1098c002f66087b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0be546d76c38468180e5687b10fa4460] 2024-11-18T03:15:31,076 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting b577a36dcd9b4f88967b288897756ad8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731899728881 2024-11-18T03:15:31,077 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:31,077 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:31,077 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,077 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b079d753b7c64f91a262f3f2bb1f24ed, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/faaa9f5fdda0432283310e52dbaeed29, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/414e10dfe41d460da8fb6dd2fa061ace] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.0 K 2024-11-18T03:15:31,077 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c260551c3d4671b1098c002f66087b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731899729283 2024-11-18T03:15:31,078 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b079d753b7c64f91a262f3f2bb1f24ed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731899728881 2024-11-18T03:15:31,078 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be546d76c38468180e5687b10fa4460, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731899729625 2024-11-18T03:15:31,079 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting faaa9f5fdda0432283310e52dbaeed29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731899729283 2024-11-18T03:15:31,080 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 414e10dfe41d460da8fb6dd2fa061ace, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731899729656 2024-11-18T03:15:31,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:31,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:31,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:31,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:31,096 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:31,101 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:31,101 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ed26a1e19e32472c9069fe61da8132f5 is 50, key is test_row_0/B:col10/1731899730789/Put/seqid=0 2024-11-18T03:15:31,105 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118fdd4c20a934640128b322c8785e14f30_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:31,107 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118fdd4c20a934640128b322c8785e14f30_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:31,107 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fdd4c20a934640128b322c8785e14f30_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742023_1199 (size=12663) 2024-11-18T03:15:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:31,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:15:31,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:31,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:31,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:31,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:31,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:31,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:31,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742024_1200 (size=4469) 2024-11-18T03:15:31,181 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ed26a1e19e32472c9069fe61da8132f5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ed26a1e19e32472c9069fe61da8132f5 2024-11-18T03:15:31,188 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into ed26a1e19e32472c9069fe61da8132f5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:31,189 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:31,189 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899731074; duration=0sec 2024-11-18T03:15:31,189 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:31,189 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:31,189 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:31,191 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:31,191 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:31,191 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,191 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/e98a5074f7d8450c9ac13ba1dd34644d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bb978f13aacc433ebc4752d2fa01f45d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/18682de21e644df5934ef0e37c95e6a4] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.0 K 2024-11-18T03:15:31,193 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e98a5074f7d8450c9ac13ba1dd34644d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731899728881 2024-11-18T03:15:31,195 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting bb978f13aacc433ebc4752d2fa01f45d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731899729283 2024-11-18T03:15:31,200 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 18682de21e644df5934ef0e37c95e6a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731899729656 2024-11-18T03:15:31,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111853266fdc44c5492f85167680c76480f0_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899731168/Put/seqid=0 2024-11-18T03:15:31,223 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:31,223 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/bcd0a697fa6641c3abffb2cf3a51af6e is 50, key is test_row_0/C:col10/1731899730789/Put/seqid=0 2024-11-18T03:15:31,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899791213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899791213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899791213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899791220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899791229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742025_1201 (size=12304) 2024-11-18T03:15:31,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:31,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111853266fdc44c5492f85167680c76480f0_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111853266fdc44c5492f85167680c76480f0_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:31,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/a420cab66d6d48a085d7633d6830a524, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:31,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/a420cab66d6d48a085d7633d6830a524 is 175, key is test_row_0/A:col10/1731899731168/Put/seqid=0 2024-11-18T03:15:31,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742026_1202 (size=12663) 2024-11-18T03:15:31,295 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/bcd0a697fa6641c3abffb2cf3a51af6e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bcd0a697fa6641c3abffb2cf3a51af6e 2024-11-18T03:15:31,301 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into bcd0a697fa6641c3abffb2cf3a51af6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:31,301 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:31,301 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899731094; duration=0sec 2024-11-18T03:15:31,301 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:31,301 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:31,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742027_1203 (size=31105) 2024-11-18T03:15:31,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/a420cab66d6d48a085d7633d6830a524 2024-11-18T03:15:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899791331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899791332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899791332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899791342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/7fe8600828b44cfb937f49e3e18379bc is 50, key is test_row_0/B:col10/1731899731168/Put/seqid=0 2024-11-18T03:15:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899791351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T03:15:31,381 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:31,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-18T03:15:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T03:15:31,385 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:31,386 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:31,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:31,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742028_1204 (size=12151) 2024-11-18T03:15:31,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/7fe8600828b44cfb937f49e3e18379bc 2024-11-18T03:15:31,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/9e07c4f4e898421f8740e791e87159d4 is 50, key is test_row_0/C:col10/1731899731168/Put/seqid=0 2024-11-18T03:15:31,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742029_1205 (size=12151) 2024-11-18T03:15:31,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/9e07c4f4e898421f8740e791e87159d4 2024-11-18T03:15:31,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/a420cab66d6d48a085d7633d6830a524 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a420cab66d6d48a085d7633d6830a524 2024-11-18T03:15:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T03:15:31,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a420cab66d6d48a085d7633d6830a524, entries=150, sequenceid=231, filesize=30.4 K 2024-11-18T03:15:31,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/7fe8600828b44cfb937f49e3e18379bc as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/7fe8600828b44cfb937f49e3e18379bc 2024-11-18T03:15:31,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/7fe8600828b44cfb937f49e3e18379bc, entries=150, sequenceid=231, filesize=11.9 K 2024-11-18T03:15:31,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/9e07c4f4e898421f8740e791e87159d4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9e07c4f4e898421f8740e791e87159d4 2024-11-18T03:15:31,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9e07c4f4e898421f8740e791e87159d4, entries=150, sequenceid=231, filesize=11.9 K 2024-11-18T03:15:31,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f0d0b02e54813e28a58a04c19edb8553 in 378ms, sequenceid=231, compaction requested=false 2024-11-18T03:15:31,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:31,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T03:15:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:31,578 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#168 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:31,579 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/a39fc7b05e3a482ebb2a374b3aec6be3 is 175, key is test_row_0/A:col10/1731899730789/Put/seqid=0 2024-11-18T03:15:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:31,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742030_1206 (size=31617) 2024-11-18T03:15:31,623 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/a39fc7b05e3a482ebb2a374b3aec6be3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a39fc7b05e3a482ebb2a374b3aec6be3 2024-11-18T03:15:31,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899791620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,632 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into a39fc7b05e3a482ebb2a374b3aec6be3(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:31,632 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:31,632 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899731073; duration=0sec 2024-11-18T03:15:31,632 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:31,632 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:31,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899791623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899791623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899791626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899791628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111807471e72981340cb99c8052612e1976a_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899731567/Put/seqid=0 2024-11-18T03:15:31,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742031_1207 (size=12304) 2024-11-18T03:15:31,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:31,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:31,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:31,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T03:15:31,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111807471e72981340cb99c8052612e1976a_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807471e72981340cb99c8052612e1976a_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:31,710 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/3bfc1e071e194f8dacd7b20751b2184e, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:31,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/3bfc1e071e194f8dacd7b20751b2184e is 175, key is test_row_0/A:col10/1731899731567/Put/seqid=0 2024-11-18T03:15:31,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742032_1208 (size=31105) 2024-11-18T03:15:31,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899791731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899791741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899791742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899791742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899791742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:31,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:31,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899791941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899791951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899791951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899791951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899791951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:32,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:32,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T03:15:32,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/3bfc1e071e194f8dacd7b20751b2184e 2024-11-18T03:15:32,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/a78a33ae39274739b9939f44c00d7ecc is 50, key is test_row_0/B:col10/1731899731567/Put/seqid=0 2024-11-18T03:15:32,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742033_1209 (size=12151) 2024-11-18T03:15:32,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899792251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899792261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899792261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899792261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899792276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T03:15:32,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/a78a33ae39274739b9939f44c00d7ecc 2024-11-18T03:15:32,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/4672dfcc44894b9f8ed4b53dfc5e8e6c is 50, key is test_row_0/C:col10/1731899731567/Put/seqid=0 2024-11-18T03:15:32,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742034_1210 (size=12151) 2024-11-18T03:15:32,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899792762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899792781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899792782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899792782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899792791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:32,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:32,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/4672dfcc44894b9f8ed4b53dfc5e8e6c 2024-11-18T03:15:32,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/3bfc1e071e194f8dacd7b20751b2184e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/3bfc1e071e194f8dacd7b20751b2184e 2024-11-18T03:15:32,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/3bfc1e071e194f8dacd7b20751b2184e, entries=150, sequenceid=250, filesize=30.4 K 2024-11-18T03:15:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/a78a33ae39274739b9939f44c00d7ecc as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/a78a33ae39274739b9939f44c00d7ecc 2024-11-18T03:15:32,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/a78a33ae39274739b9939f44c00d7ecc, entries=150, sequenceid=250, filesize=11.9 K 2024-11-18T03:15:32,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/4672dfcc44894b9f8ed4b53dfc5e8e6c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4672dfcc44894b9f8ed4b53dfc5e8e6c 2024-11-18T03:15:32,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4672dfcc44894b9f8ed4b53dfc5e8e6c, entries=150, sequenceid=250, filesize=11.9 K 2024-11-18T03:15:32,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f0d0b02e54813e28a58a04c19edb8553 in 1403ms, sequenceid=250, compaction requested=true 2024-11-18T03:15:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:32,979 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:32,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:32,980 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:32,980 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:32,981 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,981 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a39fc7b05e3a482ebb2a374b3aec6be3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a420cab66d6d48a085d7633d6830a524, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/3bfc1e071e194f8dacd7b20751b2184e] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=91.6 K 2024-11-18T03:15:32,981 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,981 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a39fc7b05e3a482ebb2a374b3aec6be3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a420cab66d6d48a085d7633d6830a524, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/3bfc1e071e194f8dacd7b20751b2184e] 2024-11-18T03:15:32,984 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:32,984 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a39fc7b05e3a482ebb2a374b3aec6be3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731899729656 2024-11-18T03:15:32,984 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:32,984 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:32,985 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ed26a1e19e32472c9069fe61da8132f5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/7fe8600828b44cfb937f49e3e18379bc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/a78a33ae39274739b9939f44c00d7ecc] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.1 K 2024-11-18T03:15:32,985 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a420cab66d6d48a085d7633d6830a524, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731899730834 2024-11-18T03:15:32,985 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ed26a1e19e32472c9069fe61da8132f5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731899729656 2024-11-18T03:15:32,985 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe8600828b44cfb937f49e3e18379bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731899730834 2024-11-18T03:15:32,986 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bfc1e071e194f8dacd7b20751b2184e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731899731213 2024-11-18T03:15:32,986 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a78a33ae39274739b9939f44c00d7ecc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731899731213 2024-11-18T03:15:32,995 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:32,997 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:32,998 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/9f9ea7c6c4c548a5869e6c0d268d0118 is 50, key is test_row_0/B:col10/1731899731567/Put/seqid=0 2024-11-18T03:15:33,009 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411187bf1bc8bcaa647b1915b1848445127a1_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:33,011 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411187bf1bc8bcaa647b1915b1848445127a1_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:33,011 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187bf1bc8bcaa647b1915b1848445127a1_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:33,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742036_1212 (size=4469) 2024-11-18T03:15:33,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742035_1211 (size=12765) 2024-11-18T03:15:33,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T03:15:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:33,082 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:15:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:33,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188356e4e16dd44669a94d42018de2f233_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899731623/Put/seqid=0 2024-11-18T03:15:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742037_1213 (size=12454) 2024-11-18T03:15:33,431 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:33,432 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/e9b65cca78f74c6a9c0b06873939fa66 is 175, key is test_row_0/A:col10/1731899731567/Put/seqid=0 2024-11-18T03:15:33,440 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/9f9ea7c6c4c548a5869e6c0d268d0118 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/9f9ea7c6c4c548a5869e6c0d268d0118 2024-11-18T03:15:33,450 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into 9f9ea7c6c4c548a5869e6c0d268d0118(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:33,450 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:33,450 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899732979; duration=0sec 2024-11-18T03:15:33,450 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:33,450 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:33,450 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:33,451 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:33,451 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:33,451 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:33,451 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bcd0a697fa6641c3abffb2cf3a51af6e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9e07c4f4e898421f8740e791e87159d4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4672dfcc44894b9f8ed4b53dfc5e8e6c] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.1 K 2024-11-18T03:15:33,452 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd0a697fa6641c3abffb2cf3a51af6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731899729656 2024-11-18T03:15:33,453 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e07c4f4e898421f8740e791e87159d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731899730834 2024-11-18T03:15:33,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742038_1214 (size=31719) 2024-11-18T03:15:33,454 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 4672dfcc44894b9f8ed4b53dfc5e8e6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731899731213 2024-11-18T03:15:33,460 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/e9b65cca78f74c6a9c0b06873939fa66 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/e9b65cca78f74c6a9c0b06873939fa66 2024-11-18T03:15:33,463 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:33,463 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/14d6203414124389ae1aef06221705a4 is 50, key is test_row_0/C:col10/1731899731567/Put/seqid=0 2024-11-18T03:15:33,467 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into e9b65cca78f74c6a9c0b06873939fa66(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:33,467 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:33,467 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899732979; duration=0sec 2024-11-18T03:15:33,467 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:33,467 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:33,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742039_1215 (size=12765) 2024-11-18T03:15:33,479 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/14d6203414124389ae1aef06221705a4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/14d6203414124389ae1aef06221705a4 2024-11-18T03:15:33,484 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into 14d6203414124389ae1aef06221705a4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:33,484 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:33,484 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899732979; duration=0sec 2024-11-18T03:15:33,484 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:33,484 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:33,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:33,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188356e4e16dd44669a94d42018de2f233_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188356e4e16dd44669a94d42018de2f233_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/897f7d379b6d40e4903a7ca2e5c607d6, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:33,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/897f7d379b6d40e4903a7ca2e5c607d6 is 175, key is test_row_0/A:col10/1731899731623/Put/seqid=0 2024-11-18T03:15:33,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742040_1216 (size=31255) 2024-11-18T03:15:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T03:15:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:33,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:33,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899793802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899793803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899793804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899793806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899793807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899793910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899793910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899793911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899793911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899793911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:33,926 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/897f7d379b6d40e4903a7ca2e5c607d6 2024-11-18T03:15:33,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ba019115a85a4c208afaf8da5d2c15ba is 50, key is test_row_0/B:col10/1731899731623/Put/seqid=0 2024-11-18T03:15:33,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742041_1217 (size=12301) 2024-11-18T03:15:34,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899794120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899794121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899794121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899794121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899794121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ba019115a85a4c208afaf8da5d2c15ba 2024-11-18T03:15:34,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a28d4c8b4eb14827bc6156102dbd4f1a is 50, key is test_row_0/C:col10/1731899731623/Put/seqid=0 2024-11-18T03:15:34,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742042_1218 (size=12301) 2024-11-18T03:15:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899794431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899794431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899794431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899794431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899794432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a28d4c8b4eb14827bc6156102dbd4f1a 2024-11-18T03:15:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/897f7d379b6d40e4903a7ca2e5c607d6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/897f7d379b6d40e4903a7ca2e5c607d6 2024-11-18T03:15:34,764 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/897f7d379b6d40e4903a7ca2e5c607d6, entries=150, sequenceid=266, filesize=30.5 K 2024-11-18T03:15:34,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ba019115a85a4c208afaf8da5d2c15ba as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ba019115a85a4c208afaf8da5d2c15ba 2024-11-18T03:15:34,770 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ba019115a85a4c208afaf8da5d2c15ba, entries=150, sequenceid=266, filesize=12.0 K 2024-11-18T03:15:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a28d4c8b4eb14827bc6156102dbd4f1a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a28d4c8b4eb14827bc6156102dbd4f1a 2024-11-18T03:15:34,776 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a28d4c8b4eb14827bc6156102dbd4f1a, entries=150, sequenceid=266, filesize=12.0 K 2024-11-18T03:15:34,777 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f0d0b02e54813e28a58a04c19edb8553 in 1695ms, sequenceid=266, compaction requested=false 2024-11-18T03:15:34,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:34,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:34,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-18T03:15:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-18T03:15:34,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-18T03:15:34,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3930 sec 2024-11-18T03:15:34,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.3980 sec 2024-11-18T03:15:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:34,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:15:34,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:34,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:34,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:34,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:34,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:34,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:34,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111841010e6342cc47e9b51eae76084e5154_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899734942/Put/seqid=0 2024-11-18T03:15:34,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899794955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899794960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899794962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899794962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:34,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899794970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742043_1219 (size=12454) 2024-11-18T03:15:35,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,007 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111841010e6342cc47e9b51eae76084e5154_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111841010e6342cc47e9b51eae76084e5154_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:35,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/77ac08ee40e34803a62034ed0643eff7, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:35,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/77ac08ee40e34803a62034ed0643eff7 is 175, key is test_row_0/A:col10/1731899734942/Put/seqid=0 2024-11-18T03:15:35,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742044_1220 (size=31255) 2024-11-18T03:15:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899795071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899795071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899795071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899795071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899795081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899795280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899795281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899795281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899795281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899795291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,439 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/77ac08ee40e34803a62034ed0643eff7 2024-11-18T03:15:35,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/d8bf430510004ae5ac8093c4585fcc3c is 50, key is test_row_0/B:col10/1731899734942/Put/seqid=0 2024-11-18T03:15:35,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742045_1221 (size=12301) 2024-11-18T03:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T03:15:35,551 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:35,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-18T03:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T03:15:35,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:35,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:35,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:35,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899795591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899795591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899795591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899795591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899795601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T03:15:35,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T03:15:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:35,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/d8bf430510004ae5ac8093c4585fcc3c 2024-11-18T03:15:35,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T03:15:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:35,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/faf2623a90d54c1abf5c66d4b85b0753 is 50, key is test_row_0/C:col10/1731899734942/Put/seqid=0 2024-11-18T03:15:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T03:15:35,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742046_1222 (size=12301) 2024-11-18T03:15:35,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/faf2623a90d54c1abf5c66d4b85b0753 2024-11-18T03:15:35,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/77ac08ee40e34803a62034ed0643eff7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/77ac08ee40e34803a62034ed0643eff7 2024-11-18T03:15:35,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/77ac08ee40e34803a62034ed0643eff7, entries=150, sequenceid=288, filesize=30.5 K 2024-11-18T03:15:35,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/d8bf430510004ae5ac8093c4585fcc3c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d8bf430510004ae5ac8093c4585fcc3c 2024-11-18T03:15:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d8bf430510004ae5ac8093c4585fcc3c, entries=150, sequenceid=288, filesize=12.0 K 2024-11-18T03:15:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/faf2623a90d54c1abf5c66d4b85b0753 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/faf2623a90d54c1abf5c66d4b85b0753 2024-11-18T03:15:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/faf2623a90d54c1abf5c66d4b85b0753, entries=150, sequenceid=288, filesize=12.0 K 2024-11-18T03:15:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for f0d0b02e54813e28a58a04c19edb8553 in 971ms, sequenceid=288, compaction requested=true 2024-11-18T03:15:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:35,914 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:35,914 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,916 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:35,916 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:35,916 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:35,916 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,916 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,916 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/e9b65cca78f74c6a9c0b06873939fa66, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/897f7d379b6d40e4903a7ca2e5c607d6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/77ac08ee40e34803a62034ed0643eff7] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=92.0 K 2024-11-18T03:15:35,916 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,916 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/e9b65cca78f74c6a9c0b06873939fa66, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/897f7d379b6d40e4903a7ca2e5c607d6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/77ac08ee40e34803a62034ed0643eff7] 2024-11-18T03:15:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,916 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,917 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/9f9ea7c6c4c548a5869e6c0d268d0118, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ba019115a85a4c208afaf8da5d2c15ba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d8bf430510004ae5ac8093c4585fcc3c] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.5 K 2024-11-18T03:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,917 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9b65cca78f74c6a9c0b06873939fa66, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731899731213 2024-11-18T03:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,918 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f9ea7c6c4c548a5869e6c0d268d0118, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731899731213 2024-11-18T03:15:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,918 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 897f7d379b6d40e4903a7ca2e5c607d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731899731621 2024-11-18T03:15:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,918 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ba019115a85a4c208afaf8da5d2c15ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731899731621 2024-11-18T03:15:35,918 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ac08ee40e34803a62034ed0643eff7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731899733805 2024-11-18T03:15:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,919 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d8bf430510004ae5ac8093c4585fcc3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731899733805 2024-11-18T03:15:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,928 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,935 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,935 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/f77e3e5d6cfa46d5aacbcdc1ae23baa1 is 50, key is test_row_0/B:col10/1731899734942/Put/seqid=0 2024-11-18T03:15:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,939 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118546fc6d5d90a414aa94f55661e8ba290_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,942 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118546fc6d5d90a414aa94f55661e8ba290_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,942 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118546fc6d5d90a414aa94f55661e8ba290_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742047_1223 (size=13017) 2024-11-18T03:15:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,977 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/f77e3e5d6cfa46d5aacbcdc1ae23baa1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f77e3e5d6cfa46d5aacbcdc1ae23baa1 2024-11-18T03:15:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,992 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into f77e3e5d6cfa46d5aacbcdc1ae23baa1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:35,992 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:35,992 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899735914; duration=0sec 2024-11-18T03:15:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,992 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:35,992 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742048_1224 (size=4469) 2024-11-18T03:15:35,992 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,993 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,993 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:35,993 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:35,994 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/14d6203414124389ae1aef06221705a4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a28d4c8b4eb14827bc6156102dbd4f1a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/faf2623a90d54c1abf5c66d4b85b0753] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.5 K 2024-11-18T03:15:35,994 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d6203414124389ae1aef06221705a4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731899731213 2024-11-18T03:15:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,994 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a28d4c8b4eb14827bc6156102dbd4f1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731899731621 2024-11-18T03:15:35,995 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting faf2623a90d54c1abf5c66d4b85b0753, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731899733805 2024-11-18T03:15:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,001 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#186 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,002 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/448a184f7c3247cbbfce51a31b624223 is 175, key is test_row_0/A:col10/1731899734942/Put/seqid=0 2024-11-18T03:15:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T03:15:36,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:36,013 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:15:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,016 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,016 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a5cfa2a02b96492aac4e5284ae8fb02c is 50, key is test_row_0/C:col10/1731899734942/Put/seqid=0 2024-11-18T03:15:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742049_1225 (size=31971) 2024-11-18T03:15:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f8585e28c510455a90d57b30322e7bd6_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899734960/Put/seqid=0 2024-11-18T03:15:36,031 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/448a184f7c3247cbbfce51a31b624223 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/448a184f7c3247cbbfce51a31b624223 2024-11-18T03:15:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,039 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into 448a184f7c3247cbbfce51a31b624223(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:36,039 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:36,039 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899735914; duration=0sec 2024-11-18T03:15:36,039 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:36,039 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742050_1226 (size=13017) 2024-11-18T03:15:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742051_1227 (size=9914) 2024-11-18T03:15:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899796146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899796150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899796152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899796154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899796154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T03:15:36,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899796261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899796261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899796271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899796271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899796272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:36,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899796472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899796472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899796482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899796482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899796487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,493 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f8585e28c510455a90d57b30322e7bd6_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f8585e28c510455a90d57b30322e7bd6_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/94cb035625294920a564d9fbc80b2844, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:36,500 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a5cfa2a02b96492aac4e5284ae8fb02c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a5cfa2a02b96492aac4e5284ae8fb02c 2024-11-18T03:15:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/94cb035625294920a564d9fbc80b2844 is 175, key is test_row_0/A:col10/1731899734960/Put/seqid=0 2024-11-18T03:15:36,528 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into a5cfa2a02b96492aac4e5284ae8fb02c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:36,528 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:36,528 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899735914; duration=0sec 2024-11-18T03:15:36,529 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:36,529 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:36,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742052_1228 (size=22561) 2024-11-18T03:15:36,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/94cb035625294920a564d9fbc80b2844 2024-11-18T03:15:36,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/cb3984f9178448c89e7b75b95163c9e1 is 50, key is test_row_0/B:col10/1731899734960/Put/seqid=0 2024-11-18T03:15:36,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742053_1229 (size=9857) 2024-11-18T03:15:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T03:15:36,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899796781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899796782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899796802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899796804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899796804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,031 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/cb3984f9178448c89e7b75b95163c9e1 2024-11-18T03:15:37,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/5b5dfc55dc90492190a393155c71d7d7 is 50, key is test_row_0/C:col10/1731899734960/Put/seqid=0 2024-11-18T03:15:37,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742054_1230 (size=9857) 2024-11-18T03:15:37,075 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/5b5dfc55dc90492190a393155c71d7d7 2024-11-18T03:15:37,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/94cb035625294920a564d9fbc80b2844 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/94cb035625294920a564d9fbc80b2844 2024-11-18T03:15:37,131 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/94cb035625294920a564d9fbc80b2844, entries=100, sequenceid=304, filesize=22.0 K 2024-11-18T03:15:37,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/cb3984f9178448c89e7b75b95163c9e1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/cb3984f9178448c89e7b75b95163c9e1 2024-11-18T03:15:37,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/cb3984f9178448c89e7b75b95163c9e1, entries=100, sequenceid=304, filesize=9.6 K 2024-11-18T03:15:37,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/5b5dfc55dc90492190a393155c71d7d7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/5b5dfc55dc90492190a393155c71d7d7 2024-11-18T03:15:37,176 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/5b5dfc55dc90492190a393155c71d7d7, entries=100, sequenceid=304, filesize=9.6 K 2024-11-18T03:15:37,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f0d0b02e54813e28a58a04c19edb8553 in 1172ms, sequenceid=304, compaction requested=false 2024-11-18T03:15:37,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:37,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:37,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-18T03:15:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-18T03:15:37,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-18T03:15:37,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6430 sec 2024-11-18T03:15:37,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6500 sec 2024-11-18T03:15:37,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:15:37,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:37,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b9cd29fec2e4cd3ab82492c825ff7c2_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899737295/Put/seqid=0 2024-11-18T03:15:37,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742055_1231 (size=17534) 2024-11-18T03:15:37,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899797347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899797348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899797354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899797355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899797359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899797464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899797468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899797471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899797472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899797476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899797681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899797682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899797691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899797691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899797691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T03:15:37,704 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:37,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-18T03:15:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T03:15:37,712 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:37,716 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:37,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:37,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:37,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b9cd29fec2e4cd3ab82492c825ff7c2_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b9cd29fec2e4cd3ab82492c825ff7c2_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:37,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/22f44848bf2244bdaf9524893024110a, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:37,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/22f44848bf2244bdaf9524893024110a is 175, key is test_row_0/A:col10/1731899737295/Put/seqid=0 2024-11-18T03:15:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742056_1232 (size=48639) 2024-11-18T03:15:37,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/22f44848bf2244bdaf9524893024110a 2024-11-18T03:15:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T03:15:37,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/77fc69bcfc2745c4b7a8699106b75c07 is 50, key is test_row_0/B:col10/1731899737295/Put/seqid=0 2024-11-18T03:15:37,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T03:15:37,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:37,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:37,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742057_1233 (size=12301) 2024-11-18T03:15:37,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/77fc69bcfc2745c4b7a8699106b75c07 2024-11-18T03:15:37,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6aa9090ec686499aad9807f39edee09b is 50, key is test_row_0/C:col10/1731899737295/Put/seqid=0 2024-11-18T03:15:37,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742058_1234 (size=12301) 2024-11-18T03:15:37,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899797991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899798001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899798001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899798001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899798002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T03:15:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T03:15:38,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T03:15:38,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:38,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T03:15:38,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T03:15:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:38,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6aa9090ec686499aad9807f39edee09b 2024-11-18T03:15:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/22f44848bf2244bdaf9524893024110a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/22f44848bf2244bdaf9524893024110a 2024-11-18T03:15:38,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/22f44848bf2244bdaf9524893024110a, entries=250, sequenceid=325, filesize=47.5 K 2024-11-18T03:15:38,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/77fc69bcfc2745c4b7a8699106b75c07 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/77fc69bcfc2745c4b7a8699106b75c07 2024-11-18T03:15:38,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/77fc69bcfc2745c4b7a8699106b75c07, entries=150, sequenceid=325, filesize=12.0 K 2024-11-18T03:15:38,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/6aa9090ec686499aad9807f39edee09b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6aa9090ec686499aad9807f39edee09b 2024-11-18T03:15:38,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6aa9090ec686499aad9807f39edee09b, entries=150, sequenceid=325, filesize=12.0 K 2024-11-18T03:15:38,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f0d0b02e54813e28a58a04c19edb8553 in 1094ms, sequenceid=325, compaction requested=true 2024-11-18T03:15:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:38,392 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:38,392 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:38,400 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:38,401 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:38,401 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,401 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:38,401 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f77e3e5d6cfa46d5aacbcdc1ae23baa1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/cb3984f9178448c89e7b75b95163c9e1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/77fc69bcfc2745c4b7a8699106b75c07] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=34.4 K 2024-11-18T03:15:38,401 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:38,401 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,401 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/448a184f7c3247cbbfce51a31b624223, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/94cb035625294920a564d9fbc80b2844, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/22f44848bf2244bdaf9524893024110a] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=100.8 K 2024-11-18T03:15:38,401 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,401 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/448a184f7c3247cbbfce51a31b624223, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/94cb035625294920a564d9fbc80b2844, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/22f44848bf2244bdaf9524893024110a] 2024-11-18T03:15:38,404 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f77e3e5d6cfa46d5aacbcdc1ae23baa1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731899733805 2024-11-18T03:15:38,404 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 448a184f7c3247cbbfce51a31b624223, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731899733805 2024-11-18T03:15:38,404 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94cb035625294920a564d9fbc80b2844, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731899734960 2024-11-18T03:15:38,404 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3984f9178448c89e7b75b95163c9e1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731899734960 2024-11-18T03:15:38,412 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f44848bf2244bdaf9524893024110a, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731899736142 2024-11-18T03:15:38,412 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 77fc69bcfc2745c4b7a8699106b75c07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731899736142 2024-11-18T03:15:38,423 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#195 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:38,424 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/e1a008949348464ea75d43611ee881a5 is 50, key is test_row_0/B:col10/1731899737295/Put/seqid=0 2024-11-18T03:15:38,433 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:38,447 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118521e2f83ef824abfa3dc2fc563477ef7_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:38,449 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118521e2f83ef824abfa3dc2fc563477ef7_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:38,449 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118521e2f83ef824abfa3dc2fc563477ef7_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:38,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742059_1235 (size=13119) 2024-11-18T03:15:38,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742060_1236 (size=4469) 2024-11-18T03:15:38,483 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/e1a008949348464ea75d43611ee881a5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e1a008949348464ea75d43611ee881a5 2024-11-18T03:15:38,483 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#196 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:38,484 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/aba951324c8449ac855de2e2ad65601f is 175, key is test_row_0/A:col10/1731899737295/Put/seqid=0 2024-11-18T03:15:38,490 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into e1a008949348464ea75d43611ee881a5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:38,490 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:38,490 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899738392; duration=0sec 2024-11-18T03:15:38,490 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:38,490 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:38,490 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:38,493 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:38,493 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:38,493 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,493 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a5cfa2a02b96492aac4e5284ae8fb02c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/5b5dfc55dc90492190a393155c71d7d7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6aa9090ec686499aad9807f39edee09b] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=34.4 K 2024-11-18T03:15:38,494 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a5cfa2a02b96492aac4e5284ae8fb02c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731899733805 2024-11-18T03:15:38,499 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b5dfc55dc90492190a393155c71d7d7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731899734960 2024-11-18T03:15:38,500 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa9090ec686499aad9807f39edee09b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731899736142 2024-11-18T03:15:38,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742061_1237 (size=32073) 2024-11-18T03:15:38,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T03:15:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:38,511 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:15:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:38,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:38,522 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:38,522 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/4e83d95608b14ccb9fa1659a6eaaf83b is 50, key is test_row_0/C:col10/1731899737295/Put/seqid=0 2024-11-18T03:15:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118608ab3df076c4e1d85d2c79b3728ba67_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899737345/Put/seqid=0 2024-11-18T03:15:38,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742062_1238 (size=13119) 2024-11-18T03:15:38,547 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/4e83d95608b14ccb9fa1659a6eaaf83b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4e83d95608b14ccb9fa1659a6eaaf83b 2024-11-18T03:15:38,558 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into 4e83d95608b14ccb9fa1659a6eaaf83b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:38,558 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:38,558 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899738392; duration=0sec 2024-11-18T03:15:38,558 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:38,558 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:38,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742063_1239 (size=12454) 2024-11-18T03:15:38,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899798548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899798550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899798552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899798554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899798558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:38,570 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118608ab3df076c4e1d85d2c79b3728ba67_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118608ab3df076c4e1d85d2c79b3728ba67_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:38,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/081a16fc3e1b47bdae102c4c4abb35fd, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:38,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/081a16fc3e1b47bdae102c4c4abb35fd is 175, key is test_row_0/A:col10/1731899737345/Put/seqid=0 2024-11-18T03:15:38,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742064_1240 (size=31255) 2024-11-18T03:15:38,589 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/081a16fc3e1b47bdae102c4c4abb35fd 2024-11-18T03:15:38,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/b1e654b844a745a38cd7ff3ff35fa1c5 is 50, key is test_row_0/B:col10/1731899737345/Put/seqid=0 2024-11-18T03:15:38,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742065_1241 (size=12301) 2024-11-18T03:15:38,625 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/b1e654b844a745a38cd7ff3ff35fa1c5 2024-11-18T03:15:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/32fef2f3ed3a4a95bfafacddcad4cf40 is 50, key is test_row_0/C:col10/1731899737345/Put/seqid=0 2024-11-18T03:15:38,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742066_1242 (size=12301) 2024-11-18T03:15:38,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899798671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899798671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899798671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899798672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899798672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T03:15:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899798892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:38,916 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/aba951324c8449ac855de2e2ad65601f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/aba951324c8449ac855de2e2ad65601f 2024-11-18T03:15:38,922 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into aba951324c8449ac855de2e2ad65601f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:38,922 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:38,922 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899738392; duration=0sec 2024-11-18T03:15:38,922 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:38,922 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:39,076 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/32fef2f3ed3a4a95bfafacddcad4cf40 2024-11-18T03:15:39,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/081a16fc3e1b47bdae102c4c4abb35fd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/081a16fc3e1b47bdae102c4c4abb35fd 2024-11-18T03:15:39,083 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/081a16fc3e1b47bdae102c4c4abb35fd, entries=150, sequenceid=339, filesize=30.5 K 2024-11-18T03:15:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/b1e654b844a745a38cd7ff3ff35fa1c5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b1e654b844a745a38cd7ff3ff35fa1c5 2024-11-18T03:15:39,088 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b1e654b844a745a38cd7ff3ff35fa1c5, entries=150, sequenceid=339, filesize=12.0 K 2024-11-18T03:15:39,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/32fef2f3ed3a4a95bfafacddcad4cf40 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/32fef2f3ed3a4a95bfafacddcad4cf40 2024-11-18T03:15:39,094 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/32fef2f3ed3a4a95bfafacddcad4cf40, entries=150, sequenceid=339, filesize=12.0 K 2024-11-18T03:15:39,095 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for f0d0b02e54813e28a58a04c19edb8553 in 584ms, sequenceid=339, compaction requested=false 2024-11-18T03:15:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-18T03:15:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-18T03:15:39,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-18T03:15:39,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3810 sec 2024-11-18T03:15:39,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.3900 sec 2024-11-18T03:15:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:39,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-18T03:15:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:39,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:39,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111892c0c1d1bff24bd1973d77c99ebbd906_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899739205/Put/seqid=0 2024-11-18T03:15:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899799210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899799211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899799211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899799212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742067_1243 (size=12454) 2024-11-18T03:15:39,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899799217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899799320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899799321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899799321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899799321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899799331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899799530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899799530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899799531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899799531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899799541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:39,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111892c0c1d1bff24bd1973d77c99ebbd906_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111892c0c1d1bff24bd1973d77c99ebbd906_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:39,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0f6f152bd63a474aa016aadbfe5489b1, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:39,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0f6f152bd63a474aa016aadbfe5489b1 is 175, key is test_row_0/A:col10/1731899739205/Put/seqid=0 2024-11-18T03:15:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742068_1244 (size=31255) 2024-11-18T03:15:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899799841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899799841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899799841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899799842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899799851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T03:15:39,871 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:39,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-18T03:15:39,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T03:15:39,874 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:39,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T03:15:40,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T03:15:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0f6f152bd63a474aa016aadbfe5489b1 2024-11-18T03:15:40,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/991dded6008847a9ac09d69e486718b6 is 50, key is test_row_0/B:col10/1731899739205/Put/seqid=0 2024-11-18T03:15:40,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742069_1245 (size=12301) 2024-11-18T03:15:40,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T03:15:40,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:40,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T03:15:40,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T03:15:40,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:40,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899800351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:40,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899800351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:40,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899800351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:40,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899800351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:40,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899800360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:40,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/991dded6008847a9ac09d69e486718b6 2024-11-18T03:15:40,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a84eadc1f1ea454096304a1fd4fde257 is 50, key is test_row_0/C:col10/1731899739205/Put/seqid=0 2024-11-18T03:15:40,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742070_1246 (size=12301) 2024-11-18T03:15:40,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T03:15:40,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:40,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T03:15:40,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T03:15:40,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:40,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T03:15:40,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:40,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:40,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a84eadc1f1ea454096304a1fd4fde257 2024-11-18T03:15:40,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/0f6f152bd63a474aa016aadbfe5489b1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0f6f152bd63a474aa016aadbfe5489b1 2024-11-18T03:15:40,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0f6f152bd63a474aa016aadbfe5489b1, entries=150, sequenceid=364, filesize=30.5 K 2024-11-18T03:15:40,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/991dded6008847a9ac09d69e486718b6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/991dded6008847a9ac09d69e486718b6 2024-11-18T03:15:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/991dded6008847a9ac09d69e486718b6, entries=150, sequenceid=364, filesize=12.0 K 2024-11-18T03:15:40,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/a84eadc1f1ea454096304a1fd4fde257 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a84eadc1f1ea454096304a1fd4fde257 2024-11-18T03:15:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a84eadc1f1ea454096304a1fd4fde257, entries=150, sequenceid=364, filesize=12.0 K 2024-11-18T03:15:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=40.25 KB/41220 for f0d0b02e54813e28a58a04c19edb8553 in 1705ms, sequenceid=364, compaction requested=true 2024-11-18T03:15:40,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,912 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:40,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,913 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,914 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:40,914 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,914 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/aba951324c8449ac855de2e2ad65601f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/081a16fc3e1b47bdae102c4c4abb35fd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0f6f152bd63a474aa016aadbfe5489b1] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=92.4 K 2024-11-18T03:15:40,914 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,914 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/aba951324c8449ac855de2e2ad65601f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/081a16fc3e1b47bdae102c4c4abb35fd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0f6f152bd63a474aa016aadbfe5489b1] 2024-11-18T03:15:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,915 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba951324c8449ac855de2e2ad65601f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731899736142 2024-11-18T03:15:40,915 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:40,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:40,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:40,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:40,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:40,916 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081a16fc3e1b47bdae102c4c4abb35fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731899737345 2024-11-18T03:15:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,917 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6f152bd63a474aa016aadbfe5489b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731899739205 2024-11-18T03:15:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,917 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:40,917 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:40,917 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,918 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e1a008949348464ea75d43611ee881a5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b1e654b844a745a38cd7ff3ff35fa1c5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/991dded6008847a9ac09d69e486718b6] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.8 K 2024-11-18T03:15:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,919 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a008949348464ea75d43611ee881a5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731899736142 2024-11-18T03:15:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,920 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e654b844a745a38cd7ff3ff35fa1c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731899737345 2024-11-18T03:15:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,922 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 991dded6008847a9ac09d69e486718b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731899739205 2024-11-18T03:15:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,936 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,947 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:40,948 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ae5f9d99f2034914b86561afa7bce543 is 50, key is test_row_0/B:col10/1731899739205/Put/seqid=0 2024-11-18T03:15:40,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T03:15:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:40,949 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-18T03:15:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,956 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411187a8ec5e9ba8240639180b6982d431040_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:40,958 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411187a8ec5e9ba8240639180b6982d431040_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:40,958 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187a8ec5e9ba8240639180b6982d431040_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742071_1247 (size=13221) 2024-11-18T03:15:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742072_1248 (size=4469) 2024-11-18T03:15:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180e394f8bf3254ddc94d504563f27909c_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899739215/Put/seqid=0 2024-11-18T03:15:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T03:15:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742073_1249 (size=9914) 2024-11-18T03:15:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,030 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180e394f8bf3254ddc94d504563f27909c_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180e394f8bf3254ddc94d504563f27909c_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/412407893a1b47d5b02598ca9cb646f7, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/412407893a1b47d5b02598ca9cb646f7 is 175, key is test_row_0/A:col10/1731899739215/Put/seqid=0 2024-11-18T03:15:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742074_1250 (size=22561) 2024-11-18T03:15:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/412407893a1b47d5b02598ca9cb646f7 2024-11-18T03:15:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/e2b6869e74cf45df9cb76ced41433ceb is 50, key is test_row_0/B:col10/1731899739215/Put/seqid=0 2024-11-18T03:15:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742075_1251 (size=9857) 2024-11-18T03:15:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,070 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/e2b6869e74cf45df9cb76ced41433ceb 2024-11-18T03:15:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/914e7087c61240a9afb80bffe5a8ac3c is 50, key is test_row_0/C:col10/1731899739215/Put/seqid=0 2024-11-18T03:15:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742076_1252 (size=9857) 2024-11-18T03:15:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:41,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. as already flushing 2024-11-18T03:15:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,381 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/ae5f9d99f2034914b86561afa7bce543 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ae5f9d99f2034914b86561afa7bce543 2024-11-18T03:15:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,389 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into ae5f9d99f2034914b86561afa7bce543(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:41,389 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:41,389 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899740913; duration=0sec 2024-11-18T03:15:41,389 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:41,389 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:41,389 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,390 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:41,390 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:41,390 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:41,390 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4e83d95608b14ccb9fa1659a6eaaf83b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/32fef2f3ed3a4a95bfafacddcad4cf40, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a84eadc1f1ea454096304a1fd4fde257] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=36.8 K 2024-11-18T03:15:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,390 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e83d95608b14ccb9fa1659a6eaaf83b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731899736142 2024-11-18T03:15:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,391 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 32fef2f3ed3a4a95bfafacddcad4cf40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731899737345 2024-11-18T03:15:41,391 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a84eadc1f1ea454096304a1fd4fde257, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731899739205 2024-11-18T03:15:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,400 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#204 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,401 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/84a179144ac74510b98db2950749e062 is 175, key is test_row_0/A:col10/1731899739205/Put/seqid=0 2024-11-18T03:15:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,404 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:41,404 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/9cb234febf8441298b46760fc5ffae92 is 50, key is test_row_0/C:col10/1731899739205/Put/seqid=0 2024-11-18T03:15:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:41,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742077_1253 (size=32175) 2024-11-18T03:15:41,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742078_1254 (size=13221) 2024-11-18T03:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899741398, max timestamp=1731899741413Num uniques 859; 2024-11-18T03:15:41,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:15:41,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899741398, max timestamp=1731899741413Num uniques 859; 2024-11-18T03:15:41,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:15:41,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899741398, max timestamp=1731899741413Num uniques 859; 2024-11-18T03:15:41,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:15:41,425 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/9cb234febf8441298b46760fc5ffae92 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9cb234febf8441298b46760fc5ffae92 2024-11-18T03:15:41,429 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into 9cb234febf8441298b46760fc5ffae92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:41,430 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:41,430 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899740915; duration=0sec 2024-11-18T03:15:41,430 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:41,430 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:41,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899801434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899801436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899801438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899801440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899801441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,494 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/914e7087c61240a9afb80bffe5a8ac3c 2024-11-18T03:15:41,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/412407893a1b47d5b02598ca9cb646f7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/412407893a1b47d5b02598ca9cb646f7 2024-11-18T03:15:41,502 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/412407893a1b47d5b02598ca9cb646f7, entries=100, sequenceid=373, filesize=22.0 K 2024-11-18T03:15:41,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/e2b6869e74cf45df9cb76ced41433ceb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e2b6869e74cf45df9cb76ced41433ceb 2024-11-18T03:15:41,507 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e2b6869e74cf45df9cb76ced41433ceb, entries=100, sequenceid=373, filesize=9.6 K 2024-11-18T03:15:41,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/914e7087c61240a9afb80bffe5a8ac3c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/914e7087c61240a9afb80bffe5a8ac3c 2024-11-18T03:15:41,513 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/914e7087c61240a9afb80bffe5a8ac3c, entries=100, sequenceid=373, filesize=9.6 K 2024-11-18T03:15:41,515 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f0d0b02e54813e28a58a04c19edb8553 in 566ms, sequenceid=373, compaction requested=false 2024-11-18T03:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-18T03:15:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-18T03:15:41,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-18T03:15:41,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6430 sec 2024-11-18T03:15:41,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6480 sec 2024-11-18T03:15:41,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=147.60 KB heapSize=416.16 KB 2024-11-18T03:15:41,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:41,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:15:41,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:41,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:15:41,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:41,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:15:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:41,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899801562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899801562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899801563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f2436dde14084cabbcfb16b2c71ade4e_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899741553/Put/seqid=0 2024-11-18T03:15:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899801570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899801571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742079_1255 (size=17534) 2024-11-18T03:15:41,634 INFO [Thread-701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-701. 2024-11-18T03:15:41,634 DEBUG [Thread-701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:41,634 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,634 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:41,636 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-18T03:15:41,636 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:41,636 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,636 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:41,637 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-18T03:15:41,637 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:41,637 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,637 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-18T03:15:41,637 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:41,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:41,637 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,637 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,637 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:41,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899801681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899801681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899801681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899801691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899801691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,821 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/84a179144ac74510b98db2950749e062 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/84a179144ac74510b98db2950749e062 2024-11-18T03:15:41,827 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into 84a179144ac74510b98db2950749e062(size=31.4 K), total size for store is 53.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:41,827 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:41,827 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899740912; duration=0sec 2024-11-18T03:15:41,827 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:41,828 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:41,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43672 deadline: 1731899801891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43688 deadline: 1731899801891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43662 deadline: 1731899801891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43712 deadline: 1731899801901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43714 deadline: 1731899801901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:15:42,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:42,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f2436dde14084cabbcfb16b2c71ade4e_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f2436dde14084cabbcfb16b2c71ade4e_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:42,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/9954a5694e8d440db52332d2312ddbdd, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:42,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/9954a5694e8d440db52332d2312ddbdd is 175, key is test_row_0/A:col10/1731899741553/Put/seqid=0 2024-11-18T03:15:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T03:15:42,021 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:42,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742080_1256 (size=48639) 2024-11-18T03:15:42,038 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/9954a5694e8d440db52332d2312ddbdd 2024-11-18T03:15:42,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/1f44aeb30cb8499aba4fc5bd9baad5a9 is 50, key is test_row_0/B:col10/1731899741553/Put/seqid=0 2024-11-18T03:15:42,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742081_1257 (size=12301) 2024-11-18T03:15:42,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/1f44aeb30cb8499aba4fc5bd9baad5a9 2024-11-18T03:15:42,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/184aefc5f5f0483784cf3ca296a3118c is 50, key is test_row_0/C:col10/1731899741553/Put/seqid=0 2024-11-18T03:15:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742082_1258 (size=12301) 2024-11-18T03:15:42,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/184aefc5f5f0483784cf3ca296a3118c 2024-11-18T03:15:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/9954a5694e8d440db52332d2312ddbdd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9954a5694e8d440db52332d2312ddbdd 2024-11-18T03:15:42,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9954a5694e8d440db52332d2312ddbdd, entries=250, sequenceid=413, filesize=47.5 K 2024-11-18T03:15:42,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/1f44aeb30cb8499aba4fc5bd9baad5a9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1f44aeb30cb8499aba4fc5bd9baad5a9 2024-11-18T03:15:42,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1f44aeb30cb8499aba4fc5bd9baad5a9, entries=150, sequenceid=413, filesize=12.0 K 2024-11-18T03:15:42,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/184aefc5f5f0483784cf3ca296a3118c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/184aefc5f5f0483784cf3ca296a3118c 2024-11-18T03:15:42,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/184aefc5f5f0483784cf3ca296a3118c, entries=150, sequenceid=413, filesize=12.0 K 2024-11-18T03:15:42,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=33.54 KB/34350 for f0d0b02e54813e28a58a04c19edb8553 in 585ms, sequenceid=413, compaction requested=true 2024-11-18T03:15:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:42,139 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:42,140 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:42,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:42,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:42,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0d0b02e54813e28a58a04c19edb8553:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:42,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:42,140 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103375 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:42,141 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/A is initiating minor compaction (all files) 2024-11-18T03:15:42,141 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:42,141 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/B is initiating minor compaction (all files) 2024-11-18T03:15:42,141 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/A in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:42,141 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/B in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:42,141 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/84a179144ac74510b98db2950749e062, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/412407893a1b47d5b02598ca9cb646f7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9954a5694e8d440db52332d2312ddbdd] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=101.0 K 2024-11-18T03:15:42,141 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ae5f9d99f2034914b86561afa7bce543, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e2b6869e74cf45df9cb76ced41433ceb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1f44aeb30cb8499aba4fc5bd9baad5a9] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=34.5 K 2024-11-18T03:15:42,141 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:42,141 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/84a179144ac74510b98db2950749e062, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/412407893a1b47d5b02598ca9cb646f7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9954a5694e8d440db52332d2312ddbdd] 2024-11-18T03:15:42,142 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a179144ac74510b98db2950749e062, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731899739205 2024-11-18T03:15:42,142 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ae5f9d99f2034914b86561afa7bce543, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731899739205 2024-11-18T03:15:42,142 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412407893a1b47d5b02598ca9cb646f7, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731899739208 2024-11-18T03:15:42,142 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b6869e74cf45df9cb76ced41433ceb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731899739208 2024-11-18T03:15:42,143 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9954a5694e8d440db52332d2312ddbdd, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731899741434 2024-11-18T03:15:42,143 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f44aeb30cb8499aba4fc5bd9baad5a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731899741439 2024-11-18T03:15:42,151 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:42,152 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/432e1d499e8e4bc4833db3dd991325a5 is 50, key is test_row_0/B:col10/1731899741553/Put/seqid=0 2024-11-18T03:15:42,163 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:42,169 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ca9560cdca8b49e4a7f1c0a44434ccc7_f0d0b02e54813e28a58a04c19edb8553 store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:42,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742083_1259 (size=13323) 2024-11-18T03:15:42,177 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/432e1d499e8e4bc4833db3dd991325a5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/432e1d499e8e4bc4833db3dd991325a5 2024-11-18T03:15:42,182 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/B of f0d0b02e54813e28a58a04c19edb8553 into 432e1d499e8e4bc4833db3dd991325a5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:42,182 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:42,182 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/B, priority=13, startTime=1731899742140; duration=0sec 2024-11-18T03:15:42,182 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:42,183 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:B 2024-11-18T03:15:42,183 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:42,184 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:42,184 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): f0d0b02e54813e28a58a04c19edb8553/C is initiating minor compaction (all files) 2024-11-18T03:15:42,184 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0d0b02e54813e28a58a04c19edb8553/C in TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:42,185 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9cb234febf8441298b46760fc5ffae92, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/914e7087c61240a9afb80bffe5a8ac3c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/184aefc5f5f0483784cf3ca296a3118c] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp, totalSize=34.5 K 2024-11-18T03:15:42,185 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb234febf8441298b46760fc5ffae92, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731899739205 2024-11-18T03:15:42,185 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 914e7087c61240a9afb80bffe5a8ac3c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731899739208 2024-11-18T03:15:42,186 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 184aefc5f5f0483784cf3ca296a3118c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731899741439 2024-11-18T03:15:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:42,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:15:42,205 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-18T03:15:42,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:42,205 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:42,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:42,205 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:42,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:42,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:42,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:42,206 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:42,206 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-18T03:15:42,206 INFO [Thread-692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-692. 2024-11-18T03:15:42,206 DEBUG [Thread-692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:42,206 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:42,206 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,206 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,206 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,206 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:42,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:42,214 INFO [Thread-690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-690. 2024-11-18T03:15:42,214 DEBUG [Thread-690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:42,214 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,214 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:42,216 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ca9560cdca8b49e4a7f1c0a44434ccc7_f0d0b02e54813e28a58a04c19edb8553, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:42,216 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-18T03:15:42,216 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:15:42,216 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,216 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ca9560cdca8b49e4a7f1c0a44434ccc7_f0d0b02e54813e28a58a04c19edb8553 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:42,216 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:42,216 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#C#compaction#215 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:42,216 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2515 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2549 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T03:15:42,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-11-18T03:15:42,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3183 rows 2024-11-18T03:15:42,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-11-18T03:15:42,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3237 rows 2024-11-18T03:15:42,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:15:42,217 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:15:42,217 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/dfac2a1b3b274e78bd73146a71b85d23 is 50, key is test_row_0/C:col10/1731899741553/Put/seqid=0 2024-11-18T03:15:42,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:42,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:15:42,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-18T03:15:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:42,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118df0b088cf64648989e376b73d8ae09dc_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_0/A:col10/1731899742203/Put/seqid=0 2024-11-18T03:15:42,223 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899742222"}]},"ts":"1731899742222"} 2024-11-18T03:15:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T03:15:42,225 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T03:15:42,225 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T03:15:42,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:15:42,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, UNASSIGN}] 2024-11-18T03:15:42,228 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, UNASSIGN 2024-11-18T03:15:42,229 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:42,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742084_1260 (size=4469) 2024-11-18T03:15:42,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742085_1261 (size=13323) 2024-11-18T03:15:42,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, UNASSIGN because future has completed 2024-11-18T03:15:42,231 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:15:42,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:15:42,233 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0d0b02e54813e28a58a04c19edb8553#A#compaction#214 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:42,234 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/d3983f6730864f96b72e0d6e4297070c is 175, key is test_row_0/A:col10/1731899741553/Put/seqid=0 2024-11-18T03:15:42,237 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/dfac2a1b3b274e78bd73146a71b85d23 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/dfac2a1b3b274e78bd73146a71b85d23 2024-11-18T03:15:42,243 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/C of f0d0b02e54813e28a58a04c19edb8553 into dfac2a1b3b274e78bd73146a71b85d23(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:42,243 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:42,243 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/C, priority=13, startTime=1731899742140; duration=0sec 2024-11-18T03:15:42,243 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:42,244 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:C 2024-11-18T03:15:42,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742086_1262 (size=12454) 2024-11-18T03:15:42,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742087_1263 (size=32277) 2024-11-18T03:15:42,263 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/d3983f6730864f96b72e0d6e4297070c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/d3983f6730864f96b72e0d6e4297070c 2024-11-18T03:15:42,268 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0d0b02e54813e28a58a04c19edb8553/A of f0d0b02e54813e28a58a04c19edb8553 into d3983f6730864f96b72e0d6e4297070c(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:42,268 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:42,268 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., storeName=f0d0b02e54813e28a58a04c19edb8553/A, priority=13, startTime=1731899742139; duration=0sec 2024-11-18T03:15:42,268 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:42,268 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0d0b02e54813e28a58a04c19edb8553:A 2024-11-18T03:15:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T03:15:42,384 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:42,384 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:15:42,385 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing f0d0b02e54813e28a58a04c19edb8553, disabling compactions & flushes 2024-11-18T03:15:42,385 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T03:15:42,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:42,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118df0b088cf64648989e376b73d8ae09dc_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118df0b088cf64648989e376b73d8ae09dc_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:42,659 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bdcddba75d7b42bcac07c9a14f631625, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:42,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bdcddba75d7b42bcac07c9a14f631625 is 175, key is test_row_0/A:col10/1731899742203/Put/seqid=0 2024-11-18T03:15:42,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742088_1264 (size=31255) 2024-11-18T03:15:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T03:15:43,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bdcddba75d7b42bcac07c9a14f631625 2024-11-18T03:15:43,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/3013a2e7c3c74ac3b0d988caf4f48898 is 50, key is test_row_0/B:col10/1731899742203/Put/seqid=0 2024-11-18T03:15:43,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742089_1265 (size=12301) 2024-11-18T03:15:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T03:15:43,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/3013a2e7c3c74ac3b0d988caf4f48898 2024-11-18T03:15:43,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/8d6eaaaf518f4daaa39697825611c29d is 50, key is test_row_0/C:col10/1731899742203/Put/seqid=0 2024-11-18T03:15:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742090_1266 (size=12301) 2024-11-18T03:15:43,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/8d6eaaaf518f4daaa39697825611c29d 2024-11-18T03:15:43,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/bdcddba75d7b42bcac07c9a14f631625 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bdcddba75d7b42bcac07c9a14f631625 2024-11-18T03:15:43,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bdcddba75d7b42bcac07c9a14f631625, entries=150, sequenceid=426, filesize=30.5 K 2024-11-18T03:15:43,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/3013a2e7c3c74ac3b0d988caf4f48898 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/3013a2e7c3c74ac3b0d988caf4f48898 2024-11-18T03:15:43,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/3013a2e7c3c74ac3b0d988caf4f48898, entries=150, sequenceid=426, filesize=12.0 K 2024-11-18T03:15:43,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/8d6eaaaf518f4daaa39697825611c29d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/8d6eaaaf518f4daaa39697825611c29d 2024-11-18T03:15:43,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/8d6eaaaf518f4daaa39697825611c29d, entries=150, sequenceid=426, filesize=12.0 K 2024-11-18T03:15:43,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for f0d0b02e54813e28a58a04c19edb8553 in 1718ms, sequenceid=426, compaction requested=false 2024-11-18T03:15:43,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0d0b02e54813e28a58a04c19edb8553: 2024-11-18T03:15:43,923 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:43,923 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:43,923 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. after waiting 0 ms 2024-11-18T03:15:43,923 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:43,924 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing f0d0b02e54813e28a58a04c19edb8553 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-18T03:15:43,924 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=A 2024-11-18T03:15:43,924 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:43,924 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=B 2024-11-18T03:15:43,924 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:43,924 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0d0b02e54813e28a58a04c19edb8553, store=C 2024-11-18T03:15:43,924 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:43,932 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118dc24eeccfa554acd9f9217687b30e038_f0d0b02e54813e28a58a04c19edb8553 is 50, key is test_row_1/A:col10/1731899742211/Put/seqid=0 2024-11-18T03:15:43,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742091_1267 (size=9914) 2024-11-18T03:15:44,337 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:44,346 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118dc24eeccfa554acd9f9217687b30e038_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dc24eeccfa554acd9f9217687b30e038_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:44,348 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/60a778081b70434e9155ec73ab629439, store: [table=TestAcidGuarantees family=A region=f0d0b02e54813e28a58a04c19edb8553] 2024-11-18T03:15:44,349 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/60a778081b70434e9155ec73ab629439 is 175, key is test_row_1/A:col10/1731899742211/Put/seqid=0 2024-11-18T03:15:44,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742092_1268 (size=22561) 2024-11-18T03:15:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T03:15:44,756 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/60a778081b70434e9155ec73ab629439 2024-11-18T03:15:44,772 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/4b95ff5917d946eb90f6e4e2105bc855 is 50, key is test_row_1/B:col10/1731899742211/Put/seqid=0 2024-11-18T03:15:44,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742093_1269 (size=9857) 2024-11-18T03:15:45,176 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/4b95ff5917d946eb90f6e4e2105bc855 2024-11-18T03:15:45,182 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/888380c7b6f448a0a97d8d6ebb58fc93 is 50, key is test_row_1/C:col10/1731899742211/Put/seqid=0 2024-11-18T03:15:45,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742094_1270 (size=9857) 2024-11-18T03:15:45,586 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/888380c7b6f448a0a97d8d6ebb58fc93 2024-11-18T03:15:45,590 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/A/60a778081b70434e9155ec73ab629439 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/60a778081b70434e9155ec73ab629439 2024-11-18T03:15:45,594 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/60a778081b70434e9155ec73ab629439, entries=100, sequenceid=433, filesize=22.0 K 2024-11-18T03:15:45,595 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/B/4b95ff5917d946eb90f6e4e2105bc855 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/4b95ff5917d946eb90f6e4e2105bc855 2024-11-18T03:15:45,599 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/4b95ff5917d946eb90f6e4e2105bc855, entries=100, sequenceid=433, filesize=9.6 K 2024-11-18T03:15:45,600 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/.tmp/C/888380c7b6f448a0a97d8d6ebb58fc93 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/888380c7b6f448a0a97d8d6ebb58fc93 2024-11-18T03:15:45,604 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/888380c7b6f448a0a97d8d6ebb58fc93, entries=100, sequenceid=433, filesize=9.6 K 2024-11-18T03:15:45,605 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f0d0b02e54813e28a58a04c19edb8553 in 1682ms, sequenceid=433, compaction requested=true 2024-11-18T03:15:45,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b41febcd8e7a4a58b4524263f69ea70b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/445ecee55d124f5397bc51c1445d0477, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/014ede28530c4a9fbb8547d46b3e1e09, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/4229e9cdef4c430b9bc66192d467b504, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2c2b238fedca4c5984496705d4bfc22d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/205667e0628c46d7af8c10a6b069cab0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bada2586ccf54cb0a8182f2305e7d394, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/f836a55e83db48a584387c18f9561c2f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/65c18ec1a74846c0948edf0fe8b5f773, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9ad6960c37ec4cbb87ef19718fa07580, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2cb938be5e0e460d9a6e5f48287142df, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/97e0e60a447c403196307947674e3538, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b577a36dcd9b4f88967b288897756ad8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/81c260551c3d4671b1098c002f66087b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0be546d76c38468180e5687b10fa4460, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a39fc7b05e3a482ebb2a374b3aec6be3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a420cab66d6d48a085d7633d6830a524, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/e9b65cca78f74c6a9c0b06873939fa66, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/3bfc1e071e194f8dacd7b20751b2184e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/897f7d379b6d40e4903a7ca2e5c607d6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/448a184f7c3247cbbfce51a31b624223, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/77ac08ee40e34803a62034ed0643eff7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/94cb035625294920a564d9fbc80b2844, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/22f44848bf2244bdaf9524893024110a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/aba951324c8449ac855de2e2ad65601f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/081a16fc3e1b47bdae102c4c4abb35fd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/84a179144ac74510b98db2950749e062, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0f6f152bd63a474aa016aadbfe5489b1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/412407893a1b47d5b02598ca9cb646f7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9954a5694e8d440db52332d2312ddbdd] to archive 2024-11-18T03:15:45,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:15:45,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b41febcd8e7a4a58b4524263f69ea70b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b41febcd8e7a4a58b4524263f69ea70b 2024-11-18T03:15:45,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/445ecee55d124f5397bc51c1445d0477 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/445ecee55d124f5397bc51c1445d0477 2024-11-18T03:15:45,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/014ede28530c4a9fbb8547d46b3e1e09 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/014ede28530c4a9fbb8547d46b3e1e09 2024-11-18T03:15:45,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/4229e9cdef4c430b9bc66192d467b504 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/4229e9cdef4c430b9bc66192d467b504 2024-11-18T03:15:45,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2c2b238fedca4c5984496705d4bfc22d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2c2b238fedca4c5984496705d4bfc22d 2024-11-18T03:15:45,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/205667e0628c46d7af8c10a6b069cab0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/205667e0628c46d7af8c10a6b069cab0 2024-11-18T03:15:45,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bada2586ccf54cb0a8182f2305e7d394 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bada2586ccf54cb0a8182f2305e7d394 2024-11-18T03:15:45,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/f836a55e83db48a584387c18f9561c2f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/f836a55e83db48a584387c18f9561c2f 2024-11-18T03:15:45,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/65c18ec1a74846c0948edf0fe8b5f773 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/65c18ec1a74846c0948edf0fe8b5f773 2024-11-18T03:15:45,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9ad6960c37ec4cbb87ef19718fa07580 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9ad6960c37ec4cbb87ef19718fa07580 2024-11-18T03:15:45,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2cb938be5e0e460d9a6e5f48287142df to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/2cb938be5e0e460d9a6e5f48287142df 2024-11-18T03:15:45,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/97e0e60a447c403196307947674e3538 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/97e0e60a447c403196307947674e3538 2024-11-18T03:15:45,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b577a36dcd9b4f88967b288897756ad8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/b577a36dcd9b4f88967b288897756ad8 2024-11-18T03:15:45,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/81c260551c3d4671b1098c002f66087b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/81c260551c3d4671b1098c002f66087b 2024-11-18T03:15:45,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0be546d76c38468180e5687b10fa4460 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0be546d76c38468180e5687b10fa4460 2024-11-18T03:15:45,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a39fc7b05e3a482ebb2a374b3aec6be3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a39fc7b05e3a482ebb2a374b3aec6be3 2024-11-18T03:15:45,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a420cab66d6d48a085d7633d6830a524 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/a420cab66d6d48a085d7633d6830a524 2024-11-18T03:15:45,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/e9b65cca78f74c6a9c0b06873939fa66 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/e9b65cca78f74c6a9c0b06873939fa66 2024-11-18T03:15:45,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/3bfc1e071e194f8dacd7b20751b2184e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/3bfc1e071e194f8dacd7b20751b2184e 2024-11-18T03:15:45,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/897f7d379b6d40e4903a7ca2e5c607d6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/897f7d379b6d40e4903a7ca2e5c607d6 2024-11-18T03:15:45,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/448a184f7c3247cbbfce51a31b624223 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/448a184f7c3247cbbfce51a31b624223 2024-11-18T03:15:45,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/77ac08ee40e34803a62034ed0643eff7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/77ac08ee40e34803a62034ed0643eff7 2024-11-18T03:15:45,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/94cb035625294920a564d9fbc80b2844 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/94cb035625294920a564d9fbc80b2844 2024-11-18T03:15:45,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/22f44848bf2244bdaf9524893024110a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/22f44848bf2244bdaf9524893024110a 2024-11-18T03:15:45,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/aba951324c8449ac855de2e2ad65601f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/aba951324c8449ac855de2e2ad65601f 2024-11-18T03:15:45,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/081a16fc3e1b47bdae102c4c4abb35fd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/081a16fc3e1b47bdae102c4c4abb35fd 2024-11-18T03:15:45,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/84a179144ac74510b98db2950749e062 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/84a179144ac74510b98db2950749e062 2024-11-18T03:15:45,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0f6f152bd63a474aa016aadbfe5489b1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/0f6f152bd63a474aa016aadbfe5489b1 2024-11-18T03:15:45,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/412407893a1b47d5b02598ca9cb646f7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/412407893a1b47d5b02598ca9cb646f7 2024-11-18T03:15:45,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9954a5694e8d440db52332d2312ddbdd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/9954a5694e8d440db52332d2312ddbdd 2024-11-18T03:15:45,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/0192dd633c2e47c2a9e9dbad8219e81f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/c91029d2754f45dcb633fc37906425b9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1dd1fd9ff87841598e0d65a413e2f7ce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/62ac982ae53f440bbe85134770c58075, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/da73b16e35924879a3c244374d7c38f9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ca0bd1ead9ca4f39b7f2aade62cf4cc8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f98af1634d79409cb4037f85c53b6003, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/8d6e5ed3225b417ba305fb101330781a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1b36751c9f33498781bb0c4d706b0003, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d326480446fd478ab4c1766e33a01456, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/88ce63ef3dd341839e3fe747b0ca469f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b079d753b7c64f91a262f3f2bb1f24ed, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/98597b277d9b4f54a2517413099eed14, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/faaa9f5fdda0432283310e52dbaeed29, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ed26a1e19e32472c9069fe61da8132f5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/414e10dfe41d460da8fb6dd2fa061ace, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/7fe8600828b44cfb937f49e3e18379bc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/9f9ea7c6c4c548a5869e6c0d268d0118, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/a78a33ae39274739b9939f44c00d7ecc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ba019115a85a4c208afaf8da5d2c15ba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f77e3e5d6cfa46d5aacbcdc1ae23baa1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d8bf430510004ae5ac8093c4585fcc3c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/cb3984f9178448c89e7b75b95163c9e1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e1a008949348464ea75d43611ee881a5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/77fc69bcfc2745c4b7a8699106b75c07, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b1e654b844a745a38cd7ff3ff35fa1c5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ae5f9d99f2034914b86561afa7bce543, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/991dded6008847a9ac09d69e486718b6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e2b6869e74cf45df9cb76ced41433ceb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1f44aeb30cb8499aba4fc5bd9baad5a9] to archive 2024-11-18T03:15:45,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:15:45,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/0192dd633c2e47c2a9e9dbad8219e81f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/0192dd633c2e47c2a9e9dbad8219e81f 2024-11-18T03:15:45,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/c91029d2754f45dcb633fc37906425b9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/c91029d2754f45dcb633fc37906425b9 2024-11-18T03:15:45,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1dd1fd9ff87841598e0d65a413e2f7ce to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1dd1fd9ff87841598e0d65a413e2f7ce 2024-11-18T03:15:45,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/62ac982ae53f440bbe85134770c58075 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/62ac982ae53f440bbe85134770c58075 2024-11-18T03:15:45,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/da73b16e35924879a3c244374d7c38f9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/da73b16e35924879a3c244374d7c38f9 2024-11-18T03:15:45,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ca0bd1ead9ca4f39b7f2aade62cf4cc8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ca0bd1ead9ca4f39b7f2aade62cf4cc8 2024-11-18T03:15:45,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f98af1634d79409cb4037f85c53b6003 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f98af1634d79409cb4037f85c53b6003 2024-11-18T03:15:45,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/8d6e5ed3225b417ba305fb101330781a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/8d6e5ed3225b417ba305fb101330781a 2024-11-18T03:15:45,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1b36751c9f33498781bb0c4d706b0003 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1b36751c9f33498781bb0c4d706b0003 2024-11-18T03:15:45,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d326480446fd478ab4c1766e33a01456 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d326480446fd478ab4c1766e33a01456 2024-11-18T03:15:45,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/88ce63ef3dd341839e3fe747b0ca469f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/88ce63ef3dd341839e3fe747b0ca469f 2024-11-18T03:15:45,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b079d753b7c64f91a262f3f2bb1f24ed to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b079d753b7c64f91a262f3f2bb1f24ed 2024-11-18T03:15:45,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/98597b277d9b4f54a2517413099eed14 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/98597b277d9b4f54a2517413099eed14 2024-11-18T03:15:45,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/faaa9f5fdda0432283310e52dbaeed29 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/faaa9f5fdda0432283310e52dbaeed29 2024-11-18T03:15:45,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ed26a1e19e32472c9069fe61da8132f5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ed26a1e19e32472c9069fe61da8132f5 2024-11-18T03:15:45,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/414e10dfe41d460da8fb6dd2fa061ace to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/414e10dfe41d460da8fb6dd2fa061ace 2024-11-18T03:15:45,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/7fe8600828b44cfb937f49e3e18379bc to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/7fe8600828b44cfb937f49e3e18379bc 2024-11-18T03:15:45,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/9f9ea7c6c4c548a5869e6c0d268d0118 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/9f9ea7c6c4c548a5869e6c0d268d0118 2024-11-18T03:15:45,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/a78a33ae39274739b9939f44c00d7ecc to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/a78a33ae39274739b9939f44c00d7ecc 2024-11-18T03:15:45,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ba019115a85a4c208afaf8da5d2c15ba to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ba019115a85a4c208afaf8da5d2c15ba 2024-11-18T03:15:45,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f77e3e5d6cfa46d5aacbcdc1ae23baa1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/f77e3e5d6cfa46d5aacbcdc1ae23baa1 2024-11-18T03:15:45,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d8bf430510004ae5ac8093c4585fcc3c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/d8bf430510004ae5ac8093c4585fcc3c 2024-11-18T03:15:45,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/cb3984f9178448c89e7b75b95163c9e1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/cb3984f9178448c89e7b75b95163c9e1 2024-11-18T03:15:45,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e1a008949348464ea75d43611ee881a5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e1a008949348464ea75d43611ee881a5 2024-11-18T03:15:45,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/77fc69bcfc2745c4b7a8699106b75c07 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/77fc69bcfc2745c4b7a8699106b75c07 2024-11-18T03:15:45,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b1e654b844a745a38cd7ff3ff35fa1c5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/b1e654b844a745a38cd7ff3ff35fa1c5 2024-11-18T03:15:45,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ae5f9d99f2034914b86561afa7bce543 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/ae5f9d99f2034914b86561afa7bce543 2024-11-18T03:15:45,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/991dded6008847a9ac09d69e486718b6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/991dded6008847a9ac09d69e486718b6 2024-11-18T03:15:45,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e2b6869e74cf45df9cb76ced41433ceb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/e2b6869e74cf45df9cb76ced41433ceb 2024-11-18T03:15:45,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1f44aeb30cb8499aba4fc5bd9baad5a9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/1f44aeb30cb8499aba4fc5bd9baad5a9 2024-11-18T03:15:45,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/898ca1615a374e249c7b77f4dbef5c9c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6a82f8e4c69542e283826aadc208b089, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/020e71f905534dbb8f348845845e9767, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6c184d724510441d847b891ba3ef53ad, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/1659a3e15df045a6b092736dcaedd7f5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/96f9e4d388e340e3a8591f993153f586, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/b9c43a6f17324419be83b68c49ab933f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/d84edc32e5fc4854a8a4396812d2339d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/cac375f78ec34236b4010a7020ba94ad, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/95dbb5609dd3401386146df9634cf0f0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/13e7408b28b2401da41c9b8a4af56734, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/e98a5074f7d8450c9ac13ba1dd34644d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/7153e1c0db094546a0115feed101826c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bb978f13aacc433ebc4752d2fa01f45d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bcd0a697fa6641c3abffb2cf3a51af6e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/18682de21e644df5934ef0e37c95e6a4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9e07c4f4e898421f8740e791e87159d4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/14d6203414124389ae1aef06221705a4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4672dfcc44894b9f8ed4b53dfc5e8e6c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a28d4c8b4eb14827bc6156102dbd4f1a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a5cfa2a02b96492aac4e5284ae8fb02c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/faf2623a90d54c1abf5c66d4b85b0753, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/5b5dfc55dc90492190a393155c71d7d7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4e83d95608b14ccb9fa1659a6eaaf83b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6aa9090ec686499aad9807f39edee09b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/32fef2f3ed3a4a95bfafacddcad4cf40, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9cb234febf8441298b46760fc5ffae92, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a84eadc1f1ea454096304a1fd4fde257, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/914e7087c61240a9afb80bffe5a8ac3c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/184aefc5f5f0483784cf3ca296a3118c] to archive 2024-11-18T03:15:45,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:15:45,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/898ca1615a374e249c7b77f4dbef5c9c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/898ca1615a374e249c7b77f4dbef5c9c 2024-11-18T03:15:45,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6a82f8e4c69542e283826aadc208b089 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6a82f8e4c69542e283826aadc208b089 2024-11-18T03:15:45,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/020e71f905534dbb8f348845845e9767 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/020e71f905534dbb8f348845845e9767 2024-11-18T03:15:45,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6c184d724510441d847b891ba3ef53ad to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6c184d724510441d847b891ba3ef53ad 2024-11-18T03:15:45,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/1659a3e15df045a6b092736dcaedd7f5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/1659a3e15df045a6b092736dcaedd7f5 2024-11-18T03:15:45,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/96f9e4d388e340e3a8591f993153f586 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/96f9e4d388e340e3a8591f993153f586 2024-11-18T03:15:45,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/b9c43a6f17324419be83b68c49ab933f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/b9c43a6f17324419be83b68c49ab933f 2024-11-18T03:15:45,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/d84edc32e5fc4854a8a4396812d2339d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/d84edc32e5fc4854a8a4396812d2339d 2024-11-18T03:15:45,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/cac375f78ec34236b4010a7020ba94ad to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/cac375f78ec34236b4010a7020ba94ad 2024-11-18T03:15:45,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/95dbb5609dd3401386146df9634cf0f0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/95dbb5609dd3401386146df9634cf0f0 2024-11-18T03:15:45,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/13e7408b28b2401da41c9b8a4af56734 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/13e7408b28b2401da41c9b8a4af56734 2024-11-18T03:15:45,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/e98a5074f7d8450c9ac13ba1dd34644d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/e98a5074f7d8450c9ac13ba1dd34644d 2024-11-18T03:15:45,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/7153e1c0db094546a0115feed101826c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/7153e1c0db094546a0115feed101826c 2024-11-18T03:15:45,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bb978f13aacc433ebc4752d2fa01f45d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bb978f13aacc433ebc4752d2fa01f45d 2024-11-18T03:15:45,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bcd0a697fa6641c3abffb2cf3a51af6e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/bcd0a697fa6641c3abffb2cf3a51af6e 2024-11-18T03:15:45,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/18682de21e644df5934ef0e37c95e6a4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/18682de21e644df5934ef0e37c95e6a4 2024-11-18T03:15:45,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9e07c4f4e898421f8740e791e87159d4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9e07c4f4e898421f8740e791e87159d4 2024-11-18T03:15:45,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/14d6203414124389ae1aef06221705a4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/14d6203414124389ae1aef06221705a4 2024-11-18T03:15:45,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4672dfcc44894b9f8ed4b53dfc5e8e6c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4672dfcc44894b9f8ed4b53dfc5e8e6c 2024-11-18T03:15:45,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a28d4c8b4eb14827bc6156102dbd4f1a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a28d4c8b4eb14827bc6156102dbd4f1a 2024-11-18T03:15:45,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a5cfa2a02b96492aac4e5284ae8fb02c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a5cfa2a02b96492aac4e5284ae8fb02c 2024-11-18T03:15:45,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/faf2623a90d54c1abf5c66d4b85b0753 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/faf2623a90d54c1abf5c66d4b85b0753 2024-11-18T03:15:45,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/5b5dfc55dc90492190a393155c71d7d7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/5b5dfc55dc90492190a393155c71d7d7 2024-11-18T03:15:45,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4e83d95608b14ccb9fa1659a6eaaf83b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/4e83d95608b14ccb9fa1659a6eaaf83b 2024-11-18T03:15:45,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6aa9090ec686499aad9807f39edee09b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/6aa9090ec686499aad9807f39edee09b 2024-11-18T03:15:45,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/32fef2f3ed3a4a95bfafacddcad4cf40 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/32fef2f3ed3a4a95bfafacddcad4cf40 2024-11-18T03:15:45,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9cb234febf8441298b46760fc5ffae92 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/9cb234febf8441298b46760fc5ffae92 2024-11-18T03:15:45,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a84eadc1f1ea454096304a1fd4fde257 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/a84eadc1f1ea454096304a1fd4fde257 2024-11-18T03:15:45,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/914e7087c61240a9afb80bffe5a8ac3c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/914e7087c61240a9afb80bffe5a8ac3c 2024-11-18T03:15:45,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/184aefc5f5f0483784cf3ca296a3118c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/184aefc5f5f0483784cf3ca296a3118c 2024-11-18T03:15:45,721 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=4 2024-11-18T03:15:45,722 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. 2024-11-18T03:15:45,722 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for f0d0b02e54813e28a58a04c19edb8553: Waiting for close lock at 1731899742385Running coprocessor pre-close hooks at 1731899742385Disabling compacts and flushes for region at 1731899742385Disabling writes for close at 1731899743923 (+1538 ms)Obtaining lock to block concurrent updates at 1731899743924 (+1 ms)Preparing flush snapshotting stores in f0d0b02e54813e28a58a04c19edb8553 at 1731899743924Finished memstore snapshotting TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731899743924Flushing stores of TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553. at 1731899743925 (+1 ms)Flushing f0d0b02e54813e28a58a04c19edb8553/A: creating writer at 1731899743925Flushing mob file f0d0b02e54813e28a58a04c19edb8553/A: appending metadata at 1731899743932 (+7 ms)Flushing mob file f0d0b02e54813e28a58a04c19edb8553/A: closing flushed file at 1731899743932Flushing f0d0b02e54813e28a58a04c19edb8553/A: appending metadata at 1731899744348 (+416 ms)Flushing f0d0b02e54813e28a58a04c19edb8553/A: closing flushed file at 1731899744348Flushing f0d0b02e54813e28a58a04c19edb8553/B: creating writer at 1731899744766 (+418 ms)Flushing f0d0b02e54813e28a58a04c19edb8553/B: appending metadata at 1731899744771 (+5 ms)Flushing f0d0b02e54813e28a58a04c19edb8553/B: closing flushed file at 1731899744771Flushing f0d0b02e54813e28a58a04c19edb8553/C: creating writer at 1731899745180 (+409 ms)Flushing f0d0b02e54813e28a58a04c19edb8553/C: appending metadata at 1731899745182 (+2 ms)Flushing f0d0b02e54813e28a58a04c19edb8553/C: closing flushed file at 1731899745182Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@730fd69: reopening flushed file at 1731899745589 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e87e06b: reopening flushed file at 1731899745594 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cc0bb8b: reopening flushed file at 1731899745599 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f0d0b02e54813e28a58a04c19edb8553 in 1682ms, sequenceid=433, compaction requested=true at 1731899745605 (+6 ms)Writing region close event to WAL at 1731899745717 (+112 ms)Running coprocessor post-close hooks at 1731899745721 (+4 ms)Closed at 1731899745722 (+1 ms) 2024-11-18T03:15:45,724 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:45,725 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f0d0b02e54813e28a58a04c19edb8553, regionState=CLOSED 2024-11-18T03:15:45,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:15:45,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T03:15:45,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure f0d0b02e54813e28a58a04c19edb8553, server=fd862c3a04f9,38549,1731899684479 in 3.4960 sec 2024-11-18T03:15:45,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-18T03:15:45,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0d0b02e54813e28a58a04c19edb8553, UNASSIGN in 3.5010 sec 2024-11-18T03:15:45,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T03:15:45,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.5040 sec 2024-11-18T03:15:45,733 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899745733"}]},"ts":"1731899745733"} 2024-11-18T03:15:45,734 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T03:15:45,734 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T03:15:45,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.5170 sec 2024-11-18T03:15:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T03:15:46,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:46,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-18T03:15:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:46,389 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T03:15:46,390 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:46,395 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,399 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/recovered.edits] 2024-11-18T03:15:46,403 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/60a778081b70434e9155ec73ab629439 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/60a778081b70434e9155ec73ab629439 2024-11-18T03:15:46,404 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bdcddba75d7b42bcac07c9a14f631625 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/bdcddba75d7b42bcac07c9a14f631625 2024-11-18T03:15:46,406 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/d3983f6730864f96b72e0d6e4297070c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/A/d3983f6730864f96b72e0d6e4297070c 2024-11-18T03:15:46,408 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/3013a2e7c3c74ac3b0d988caf4f48898 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/3013a2e7c3c74ac3b0d988caf4f48898 2024-11-18T03:15:46,410 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/432e1d499e8e4bc4833db3dd991325a5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/432e1d499e8e4bc4833db3dd991325a5 2024-11-18T03:15:46,411 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/4b95ff5917d946eb90f6e4e2105bc855 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/B/4b95ff5917d946eb90f6e4e2105bc855 2024-11-18T03:15:46,413 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/888380c7b6f448a0a97d8d6ebb58fc93 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/888380c7b6f448a0a97d8d6ebb58fc93 2024-11-18T03:15:46,415 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/8d6eaaaf518f4daaa39697825611c29d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/8d6eaaaf518f4daaa39697825611c29d 2024-11-18T03:15:46,416 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/dfac2a1b3b274e78bd73146a71b85d23 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/C/dfac2a1b3b274e78bd73146a71b85d23 2024-11-18T03:15:46,419 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/recovered.edits/436.seqid to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553/recovered.edits/436.seqid 2024-11-18T03:15:46,420 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,420 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T03:15:46,420 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T03:15:46,421 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T03:15:46,426 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111801fd7ca647b14fa0b86c341914901206_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111801fd7ca647b14fa0b86c341914901206_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,427 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807471e72981340cb99c8052612e1976a_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807471e72981340cb99c8052612e1976a_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,429 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180e394f8bf3254ddc94d504563f27909c_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180e394f8bf3254ddc94d504563f27909c_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,430 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118368f706011a24ce18cc80169214ea74f_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118368f706011a24ce18cc80169214ea74f_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,432 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111841010e6342cc47e9b51eae76084e5154_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111841010e6342cc47e9b51eae76084e5154_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,434 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118445b476e16e944b9bd7e1204f1b5c1af_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118445b476e16e944b9bd7e1204f1b5c1af_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,435 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118478c9ae7acb34b29961c14a7f63cc8bf_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118478c9ae7acb34b29961c14a7f63cc8bf_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,437 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111853266fdc44c5492f85167680c76480f0_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111853266fdc44c5492f85167680c76480f0_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,439 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118608ab3df076c4e1d85d2c79b3728ba67_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118608ab3df076c4e1d85d2c79b3728ba67_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,440 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186c3e2482ec6a45038edbe3e06691defa_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186c3e2482ec6a45038edbe3e06691defa_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,442 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b9cd29fec2e4cd3ab82492c825ff7c2_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b9cd29fec2e4cd3ab82492c825ff7c2_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,444 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188356e4e16dd44669a94d42018de2f233_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188356e4e16dd44669a94d42018de2f233_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,446 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111892c0c1d1bff24bd1973d77c99ebbd906_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111892c0c1d1bff24bd1973d77c99ebbd906_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,448 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b3816c9b080344c09c594281214389d7_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b3816c9b080344c09c594281214389d7_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,449 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cd282be6e9cb4d958bd399ba887507e3_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cd282be6e9cb4d958bd399ba887507e3_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,451 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dc24eeccfa554acd9f9217687b30e038_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dc24eeccfa554acd9f9217687b30e038_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,452 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118df0b088cf64648989e376b73d8ae09dc_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118df0b088cf64648989e376b73d8ae09dc_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,454 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e01c2c231d28457b94c3b57d5c4f4d8c_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e01c2c231d28457b94c3b57d5c4f4d8c_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,455 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e0af5119555b4403937c40ecf0910a4b_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e0af5119555b4403937c40ecf0910a4b_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,456 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee065d1795a5408e9e5ba449f8b2a8d3_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee065d1795a5408e9e5ba449f8b2a8d3_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,457 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f2436dde14084cabbcfb16b2c71ade4e_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f2436dde14084cabbcfb16b2c71ade4e_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,458 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f8585e28c510455a90d57b30322e7bd6_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f8585e28c510455a90d57b30322e7bd6_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,459 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ff14fa6549a54d268d03af05bd3166d9_f0d0b02e54813e28a58a04c19edb8553 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ff14fa6549a54d268d03af05bd3166d9_f0d0b02e54813e28a58a04c19edb8553 2024-11-18T03:15:46,460 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T03:15:46,462 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:46,464 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T03:15:46,466 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T03:15:46,467 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:46,467 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T03:15:46,467 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731899746467"}]},"ts":"9223372036854775807"} 2024-11-18T03:15:46,469 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T03:15:46,469 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f0d0b02e54813e28a58a04c19edb8553, NAME => 'TestAcidGuarantees,,1731899718354.f0d0b02e54813e28a58a04c19edb8553.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T03:15:46,469 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T03:15:46,469 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731899746469"}]},"ts":"9223372036854775807"} 2024-11-18T03:15:46,471 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T03:15:46,471 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:46,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-11-18T03:15:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T03:15:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T03:15:46,501 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:46,514 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 250) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2140850966_22 at /127.0.0.1:54996 [Waiting for operation #859] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_114200576_22 at /127.0.0.1:55238 [Waiting for operation #712] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_114200576_22 at /127.0.0.1:55020 [Waiting for operation #799] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2140850966_22 at /127.0.0.1:41002 [Waiting for operation #877] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=789 (was 871), ProcessCount=11 (was 11), AvailableMemoryMB=3264 (was 3009) - AvailableMemoryMB LEAK? - 2024-11-18T03:15:46,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=789, ProcessCount=11, AvailableMemoryMB=3263 2024-11-18T03:15:46,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:15:46,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:15:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T03:15:46,528 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:15:46,528 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:46,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-18T03:15:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T03:15:46,529 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:15:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742095_1271 (size=1061) 2024-11-18T03:15:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T03:15:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T03:15:46,939 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:15:46,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742096_1272 (size=53) 2024-11-18T03:15:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T03:15:47,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:15:47,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 796efe4cc831df7448a6f85d3ad0d44b, disabling compactions & flushes 2024-11-18T03:15:47,350 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. after waiting 0 ms 2024-11-18T03:15:47,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,350 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 796efe4cc831df7448a6f85d3ad0d44b: Waiting for close lock at 1731899747350Disabling compacts and flushes for region at 1731899747350Disabling writes for close at 1731899747350Writing region close event to WAL at 1731899747350Closed at 1731899747350 2024-11-18T03:15:47,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:15:47,352 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731899747352"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731899747352"}]},"ts":"1731899747352"} 2024-11-18T03:15:47,354 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:15:47,356 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:15:47,356 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899747356"}]},"ts":"1731899747356"} 2024-11-18T03:15:47,358 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T03:15:47,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, ASSIGN}] 2024-11-18T03:15:47,360 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, ASSIGN 2024-11-18T03:15:47,361 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, ASSIGN; state=OFFLINE, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=false 2024-11-18T03:15:47,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=796efe4cc831df7448a6f85d3ad0d44b, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:47,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, ASSIGN because future has completed 2024-11-18T03:15:47,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T03:15:47,672 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,673 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:15:47,673 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,674 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:15:47,674 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,674 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,676 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,678 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:47,679 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 796efe4cc831df7448a6f85d3ad0d44b columnFamilyName A 2024-11-18T03:15:47,679 DEBUG [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:47,680 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.HStore(327): Store=796efe4cc831df7448a6f85d3ad0d44b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:47,680 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,682 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:47,682 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 796efe4cc831df7448a6f85d3ad0d44b columnFamilyName B 2024-11-18T03:15:47,682 DEBUG [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:47,683 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.HStore(327): Store=796efe4cc831df7448a6f85d3ad0d44b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:47,683 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,684 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:15:47,685 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 796efe4cc831df7448a6f85d3ad0d44b columnFamilyName C 2024-11-18T03:15:47,685 DEBUG [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:15:47,685 INFO [StoreOpener-796efe4cc831df7448a6f85d3ad0d44b-1 {}] regionserver.HStore(327): Store=796efe4cc831df7448a6f85d3ad0d44b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:15:47,685 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,685 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,686 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,687 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,688 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,688 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,688 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:15:47,690 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,692 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:15:47,692 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 796efe4cc831df7448a6f85d3ad0d44b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58820198, jitterRate=-0.12351074814796448}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:15:47,692 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:47,693 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 796efe4cc831df7448a6f85d3ad0d44b: Running coprocessor pre-open hook at 1731899747674Writing region info on filesystem at 1731899747674Initializing all the Stores at 1731899747675 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899747675Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899747676 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899747676Cleaning up temporary data from old regions at 1731899747688 (+12 ms)Running coprocessor post-open hooks at 1731899747692 (+4 ms)Region opened successfully at 1731899747693 (+1 ms) 2024-11-18T03:15:47,694 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., pid=65, masterSystemTime=1731899747666 2024-11-18T03:15:47,696 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,696 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:47,697 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=796efe4cc831df7448a6f85d3ad0d44b, regionState=OPEN, openSeqNum=2, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:47,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:15:47,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-18T03:15:47,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 in 185 msec 2024-11-18T03:15:47,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-18T03:15:47,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, ASSIGN in 342 msec 2024-11-18T03:15:47,702 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:15:47,703 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899747702"}]},"ts":"1731899747702"} 2024-11-18T03:15:47,704 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T03:15:47,704 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:15:47,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-18T03:15:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T03:15:48,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:48,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc110cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f28d93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17824b2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,679 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,683 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fd862c3a04f9,45051,1731899683325 2024-11-18T03:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@680d835d 2024-11-18T03:15:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:15:48,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:15:48,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acdcd11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,689 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53287f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,692 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4195463b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6008ec06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,695 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d19181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,697 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3206e600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422d83a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,699 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f14318c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68dcedc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a060d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,703 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746e1565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,705 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bec4753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be8a148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,707 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243158ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db86d20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf168e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,711 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3be20bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a68e01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a08f339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,714 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b68bb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4008bd06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab4c7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f174051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,718 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a9df9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38408430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,720 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ef65f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,721 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39103d86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b958231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:15:48,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:15:48,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:15:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:15:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:15:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bef4340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:15:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:15:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:15:48,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:15:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195c7d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:15:48,726 DEBUG [Thread-1255 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,726 DEBUG [Thread-1249 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,726 DEBUG [Thread-1251 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,726 DEBUG [Thread-1253 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,727 DEBUG [Thread-1247 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,727 DEBUG [Thread-1266 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,727 DEBUG [Thread-1262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,727 DEBUG [Thread-1260 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-18T03:15:48,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,729 DEBUG [Thread-1264 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,730 DEBUG [Thread-1258 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:15:48,730 DEBUG [Thread-1262 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,730 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:48,731 DEBUG [Thread-1262 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T03:15:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:15:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:15:48,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,733 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:48,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:15:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:15:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:48,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:15:48,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:48,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/79c76560dfe94db1b7b0170143313cf4 is 50, key is test_row_0/A:col10/1731899748735/Put/seqid=0 2024-11-18T03:15:48,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742097_1273 (size=12001) 2024-11-18T03:15:48,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899748755, max timestamp=1731899748796Num uniques 859; 2024-11-18T03:15:48,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899748755, max timestamp=1731899748796Num uniques 859; 2024-11-18T03:15:48,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:48,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:48,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899748755, max timestamp=1731899748796Num uniques 859; 2024-11-18T03:15:48,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:48,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899808822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899808825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899808825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899808826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899808827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T03:15:48,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:48,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:48,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899808931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899808940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899808940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899808941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899808941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T03:15:49,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899809141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899809151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899809151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899809151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899809151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/79c76560dfe94db1b7b0170143313cf4 2024-11-18T03:15:49,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/43ce3261582b48268be2305579f3ce76 is 50, key is test_row_0/B:col10/1731899748735/Put/seqid=0 2024-11-18T03:15:49,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742098_1274 (size=12001) 2024-11-18T03:15:49,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:49,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:49,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T03:15:49,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899809451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899809461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899809461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899809461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899809461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/43ce3261582b48268be2305579f3ce76 2024-11-18T03:15:49,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:49,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:49,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/f3ef7a79d990401bac4d7386f94cb288 is 50, key is test_row_0/C:col10/1731899748735/Put/seqid=0 2024-11-18T03:15:49,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742099_1275 (size=12001) 2024-11-18T03:15:49,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T03:15:49,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:49,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:49,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:49,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:49,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899809962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899809971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899809972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899809972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:49,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899809972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:50,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/f3ef7a79d990401bac4d7386f94cb288 2024-11-18T03:15:50,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/79c76560dfe94db1b7b0170143313cf4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/79c76560dfe94db1b7b0170143313cf4 2024-11-18T03:15:50,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/79c76560dfe94db1b7b0170143313cf4, entries=150, sequenceid=11, filesize=11.7 K 2024-11-18T03:15:50,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/43ce3261582b48268be2305579f3ce76 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/43ce3261582b48268be2305579f3ce76 2024-11-18T03:15:50,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/43ce3261582b48268be2305579f3ce76, entries=150, sequenceid=11, filesize=11.7 K 2024-11-18T03:15:50,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/f3ef7a79d990401bac4d7386f94cb288 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f3ef7a79d990401bac4d7386f94cb288 2024-11-18T03:15:50,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f3ef7a79d990401bac4d7386f94cb288, entries=150, sequenceid=11, filesize=11.7 K 2024-11-18T03:15:50,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 796efe4cc831df7448a6f85d3ad0d44b in 1347ms, sequenceid=11, compaction requested=false 2024-11-18T03:15:50,087 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T03:15:50,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:50,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T03:15:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:50,117 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T03:15:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/e576b933e8b54366ac5244e586c4a021 is 50, key is test_row_0/A:col10/1731899748825/Put/seqid=0 2024-11-18T03:15:50,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742100_1276 (size=12001) 2024-11-18T03:15:50,567 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/e576b933e8b54366ac5244e586c4a021 2024-11-18T03:15:50,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/faf5fae0927446ef93f1563f3e16e2bb is 50, key is test_row_0/B:col10/1731899748825/Put/seqid=0 2024-11-18T03:15:50,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742101_1277 (size=12001) 2024-11-18T03:15:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T03:15:51,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899811028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899811035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899811038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899811039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899811041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,077 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/faf5fae0927446ef93f1563f3e16e2bb 2024-11-18T03:15:51,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/39edec4daca342ea8791bd9952bb85c8 is 50, key is test_row_0/C:col10/1731899748825/Put/seqid=0 2024-11-18T03:15:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742102_1278 (size=12001) 2024-11-18T03:15:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899811141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899811152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899811152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899811152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899811152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:15:51,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899811355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899811361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899811361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899811371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899811372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/39edec4daca342ea8791bd9952bb85c8 2024-11-18T03:15:51,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/e576b933e8b54366ac5244e586c4a021 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/e576b933e8b54366ac5244e586c4a021 2024-11-18T03:15:51,567 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/e576b933e8b54366ac5244e586c4a021, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T03:15:51,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/faf5fae0927446ef93f1563f3e16e2bb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faf5fae0927446ef93f1563f3e16e2bb 2024-11-18T03:15:51,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faf5fae0927446ef93f1563f3e16e2bb, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T03:15:51,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/39edec4daca342ea8791bd9952bb85c8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/39edec4daca342ea8791bd9952bb85c8 2024-11-18T03:15:51,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/39edec4daca342ea8791bd9952bb85c8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T03:15:51,597 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=60.38 KB/61830 for 796efe4cc831df7448a6f85d3ad0d44b in 1480ms, sequenceid=48, compaction requested=false 2024-11-18T03:15:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-18T03:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-18T03:15:51,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-18T03:15:51,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8660 sec 2024-11-18T03:15:51,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.8760 sec 2024-11-18T03:15:51,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:15:51,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:51,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:51,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:51,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:51,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:51,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:51,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/3ea5801bca4b4b33aa9b81503022d303 is 50, key is test_row_0/A:col10/1731899751035/Put/seqid=0 2024-11-18T03:15:51,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742103_1279 (size=12001) 2024-11-18T03:15:51,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/3ea5801bca4b4b33aa9b81503022d303 2024-11-18T03:15:51,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899811708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899811711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899811717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/7d7c3a54c5734718bbbf2d356b6f2ec6 is 50, key is test_row_0/B:col10/1731899751035/Put/seqid=0 2024-11-18T03:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899811717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899811719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742104_1280 (size=12001) 2024-11-18T03:15:51,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/7d7c3a54c5734718bbbf2d356b6f2ec6 2024-11-18T03:15:51,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/fe076de632e84f82a8ebde64da3f2d82 is 50, key is test_row_0/C:col10/1731899751035/Put/seqid=0 2024-11-18T03:15:51,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742105_1281 (size=12001) 2024-11-18T03:15:51,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/fe076de632e84f82a8ebde64da3f2d82 2024-11-18T03:15:51,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/3ea5801bca4b4b33aa9b81503022d303 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/3ea5801bca4b4b33aa9b81503022d303 2024-11-18T03:15:51,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/3ea5801bca4b4b33aa9b81503022d303, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T03:15:51,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/7d7c3a54c5734718bbbf2d356b6f2ec6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/7d7c3a54c5734718bbbf2d356b6f2ec6 2024-11-18T03:15:51,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/7d7c3a54c5734718bbbf2d356b6f2ec6, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T03:15:51,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/fe076de632e84f82a8ebde64da3f2d82 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/fe076de632e84f82a8ebde64da3f2d82 2024-11-18T03:15:51,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899811821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899811821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899811831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899811831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:51,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/fe076de632e84f82a8ebde64da3f2d82, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T03:15:51,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 796efe4cc831df7448a6f85d3ad0d44b in 161ms, sequenceid=61, compaction requested=true 2024-11-18T03:15:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:51,836 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:51,836 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:51,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:51,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:51,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:51,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:51,838 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:51,838 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:15:51,838 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:51,838 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/79c76560dfe94db1b7b0170143313cf4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/e576b933e8b54366ac5244e586c4a021, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/3ea5801bca4b4b33aa9b81503022d303] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=35.2 K 2024-11-18T03:15:51,838 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79c76560dfe94db1b7b0170143313cf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731899748735 2024-11-18T03:15:51,839 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e576b933e8b54366ac5244e586c4a021, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731899748823 2024-11-18T03:15:51,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899751708, max timestamp=1731899751836Num uniques 859; 2024-11-18T03:15:51,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899751708, max timestamp=1731899751836Num uniques 859; 2024-11-18T03:15:51,839 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ea5801bca4b4b33aa9b81503022d303, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731899751035 2024-11-18T03:15:51,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:51,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:51,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-18T03:15:51,840 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:51,840 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:15:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:51,840 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:51,840 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/43ce3261582b48268be2305579f3ce76, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faf5fae0927446ef93f1563f3e16e2bb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/7d7c3a54c5734718bbbf2d356b6f2ec6] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=35.2 K 2024-11-18T03:15:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:51,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:51,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:51,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:51,843 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ce3261582b48268be2305579f3ce76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731899748735 2024-11-18T03:15:51,843 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting faf5fae0927446ef93f1563f3e16e2bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731899748823 2024-11-18T03:15:51,844 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7c3a54c5734718bbbf2d356b6f2ec6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731899751035 2024-11-18T03:15:51,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/962ab15d5cab4391ba172ab60f2af322 is 50, key is test_row_0/A:col10/1731899751708/Put/seqid=0 2024-11-18T03:15:51,865 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:51,866 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/26e7e16602d84a1287ec1fe1647f1409 is 50, key is test_row_0/B:col10/1731899751035/Put/seqid=0 2024-11-18T03:15:51,874 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:51,875 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/d81e4c4be5894267a16745656069bae8 is 50, key is test_row_0/A:col10/1731899751035/Put/seqid=0 2024-11-18T03:15:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742107_1283 (size=12104) 2024-11-18T03:15:51,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742108_1284 (size=12104) 2024-11-18T03:15:51,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742106_1282 (size=12001) 2024-11-18T03:15:51,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/962ab15d5cab4391ba172ab60f2af322 2024-11-18T03:15:51,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2 is 50, key is test_row_0/B:col10/1731899751708/Put/seqid=0 2024-11-18T03:15:51,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742109_1285 (size=12001) 2024-11-18T03:15:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899751892, max timestamp=1731899751954Num uniques 859; 2024-11-18T03:15:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899751892, max timestamp=1731899751954Num uniques 859; 2024-11-18T03:15:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899751892, max timestamp=1731899751954Num uniques 859; 2024-11-18T03:15:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:52,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899812036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899812036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899812039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899812041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899812041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899812151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899812151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899812151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,302 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/26e7e16602d84a1287ec1fe1647f1409 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/26e7e16602d84a1287ec1fe1647f1409 2024-11-18T03:15:52,307 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 26e7e16602d84a1287ec1fe1647f1409(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:52,307 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:52,307 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899751836; duration=0sec 2024-11-18T03:15:52,307 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:52,307 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:15:52,307 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:52,308 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:52,308 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:15:52,308 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:52,308 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f3ef7a79d990401bac4d7386f94cb288, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/39edec4daca342ea8791bd9952bb85c8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/fe076de632e84f82a8ebde64da3f2d82] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=35.2 K 2024-11-18T03:15:52,309 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ef7a79d990401bac4d7386f94cb288, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731899748735 2024-11-18T03:15:52,310 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 39edec4daca342ea8791bd9952bb85c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731899748823 2024-11-18T03:15:52,311 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting fe076de632e84f82a8ebde64da3f2d82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731899751035 2024-11-18T03:15:52,311 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/d81e4c4be5894267a16745656069bae8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d81e4c4be5894267a16745656069bae8 2024-11-18T03:15:52,316 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into d81e4c4be5894267a16745656069bae8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:52,316 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:52,316 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899751836; duration=0sec 2024-11-18T03:15:52,316 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:52,316 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:15:52,319 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:52,320 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7d53b46c95f44d7496d090cd0d612de2 is 50, key is test_row_0/C:col10/1731899751035/Put/seqid=0 2024-11-18T03:15:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742110_1286 (size=12104) 2024-11-18T03:15:52,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2 2024-11-18T03:15:52,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899812351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899812351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/aeda347f7fb64522b39b618bc1b39062 is 50, key is test_row_0/C:col10/1731899751708/Put/seqid=0 2024-11-18T03:15:52,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742111_1287 (size=12001) 2024-11-18T03:15:52,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/aeda347f7fb64522b39b618bc1b39062 2024-11-18T03:15:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899812360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899812361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899812361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/962ab15d5cab4391ba172ab60f2af322 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/962ab15d5cab4391ba172ab60f2af322 2024-11-18T03:15:52,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/962ab15d5cab4391ba172ab60f2af322, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T03:15:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2 2024-11-18T03:15:52,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T03:15:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/aeda347f7fb64522b39b618bc1b39062 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/aeda347f7fb64522b39b618bc1b39062 2024-11-18T03:15:52,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/aeda347f7fb64522b39b618bc1b39062, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T03:15:52,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 796efe4cc831df7448a6f85d3ad0d44b in 548ms, sequenceid=82, compaction requested=false 2024-11-18T03:15:52,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:52,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T03:15:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:52,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/84e89e52dccd4323b04335cc422bcbfa is 50, key is test_row_0/A:col10/1731899752013/Put/seqid=0 2024-11-18T03:15:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742112_1288 (size=14341) 2024-11-18T03:15:52,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/84e89e52dccd4323b04335cc422bcbfa 2024-11-18T03:15:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899812682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/c719f3e31a354b72b1511e0c0ec897ac is 50, key is test_row_0/B:col10/1731899752013/Put/seqid=0 2024-11-18T03:15:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742113_1289 (size=12001) 2024-11-18T03:15:52,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899812688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899812689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,730 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7d53b46c95f44d7496d090cd0d612de2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7d53b46c95f44d7496d090cd0d612de2 2024-11-18T03:15:52,734 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into 7d53b46c95f44d7496d090cd0d612de2(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:52,734 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:52,734 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899751837; duration=0sec 2024-11-18T03:15:52,734 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:52,734 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:15:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899812801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899812801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899812801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899812861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899812861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T03:15:52,892 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:52,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-18T03:15:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T03:15:52,893 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:52,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:52,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T03:15:53,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899813012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899813012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899813012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T03:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/c719f3e31a354b72b1511e0c0ec897ac 2024-11-18T03:15:53,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/af30865c3b0646edbc4c39ef10ab469d is 50, key is test_row_0/C:col10/1731899752013/Put/seqid=0 2024-11-18T03:15:53,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742114_1290 (size=12001) 2024-11-18T03:15:53,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T03:15:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T03:15:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899813321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899813321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899813321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T03:15:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/af30865c3b0646edbc4c39ef10ab469d 2024-11-18T03:15:53,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T03:15:53,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:53,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:53,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/84e89e52dccd4323b04335cc422bcbfa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/84e89e52dccd4323b04335cc422bcbfa 2024-11-18T03:15:53,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/84e89e52dccd4323b04335cc422bcbfa, entries=200, sequenceid=121, filesize=14.0 K 2024-11-18T03:15:53,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/c719f3e31a354b72b1511e0c0ec897ac as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c719f3e31a354b72b1511e0c0ec897ac 2024-11-18T03:15:53,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c719f3e31a354b72b1511e0c0ec897ac, entries=150, sequenceid=121, filesize=11.7 K 2024-11-18T03:15:53,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/af30865c3b0646edbc4c39ef10ab469d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/af30865c3b0646edbc4c39ef10ab469d 2024-11-18T03:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T03:15:53,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/af30865c3b0646edbc4c39ef10ab469d, entries=150, sequenceid=121, filesize=11.7 K 2024-11-18T03:15:53,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 796efe4cc831df7448a6f85d3ad0d44b in 849ms, sequenceid=121, compaction requested=true 2024-11-18T03:15:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:53,523 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:53,523 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:53,524 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:53,524 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:53,524 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:15:53,524 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:15:53,524 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,524 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,524 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/26e7e16602d84a1287ec1fe1647f1409, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c719f3e31a354b72b1511e0c0ec897ac] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=35.3 K 2024-11-18T03:15:53,524 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d81e4c4be5894267a16745656069bae8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/962ab15d5cab4391ba172ab60f2af322, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/84e89e52dccd4323b04335cc422bcbfa] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=37.5 K 2024-11-18T03:15:53,525 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d81e4c4be5894267a16745656069bae8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731899751035 2024-11-18T03:15:53,525 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e7e16602d84a1287ec1fe1647f1409, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731899751035 2024-11-18T03:15:53,525 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ec6649f5d35b4b1b9ca5a2f4c8e377e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731899751708 2024-11-18T03:15:53,525 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962ab15d5cab4391ba172ab60f2af322, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731899751708 2024-11-18T03:15:53,525 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c719f3e31a354b72b1511e0c0ec897ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731899752013 2024-11-18T03:15:53,525 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e89e52dccd4323b04335cc422bcbfa, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731899752013 2024-11-18T03:15:53,558 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:53,558 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:53,559 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/6becb7156309408fb3891b3bdd3dfe00 is 50, key is test_row_0/A:col10/1731899752013/Put/seqid=0 2024-11-18T03:15:53,559 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/291bff8b2966410381adae89b4851ec9 is 50, key is test_row_0/B:col10/1731899752013/Put/seqid=0 2024-11-18T03:15:53,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742115_1291 (size=12207) 2024-11-18T03:15:53,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742116_1292 (size=12207) 2024-11-18T03:15:53,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T03:15:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:15:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/1f4050faec694d1baa97364e8a975761 is 50, key is test_row_0/A:col10/1731899752688/Put/seqid=0 2024-11-18T03:15:53,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742117_1293 (size=12051) 2024-11-18T03:15:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:53,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899753857, max timestamp=1731899753866Num uniques 859; 2024-11-18T03:15:53,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:15:53,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899753857, max timestamp=1731899753866Num uniques 859; 2024-11-18T03:15:53,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:15:53,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899753857, max timestamp=1731899753866Num uniques 859; 2024-11-18T03:15:53,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:15:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899813886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899813887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899813893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899813894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899813895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:53,977 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/6becb7156309408fb3891b3bdd3dfe00 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6becb7156309408fb3891b3bdd3dfe00 2024-11-18T03:15:53,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/291bff8b2966410381adae89b4851ec9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/291bff8b2966410381adae89b4851ec9 2024-11-18T03:15:53,983 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into 6becb7156309408fb3891b3bdd3dfe00(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:53,983 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:53,983 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899753523; duration=0sec 2024-11-18T03:15:53,983 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:53,983 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:15:53,983 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:53,984 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 291bff8b2966410381adae89b4851ec9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:53,984 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:53,984 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:53,984 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:15:53,984 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899753523; duration=0sec 2024-11-18T03:15:53,984 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:53,984 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:53,984 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:15:53,984 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7d53b46c95f44d7496d090cd0d612de2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/aeda347f7fb64522b39b618bc1b39062, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/af30865c3b0646edbc4c39ef10ab469d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=35.3 K 2024-11-18T03:15:53,985 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d53b46c95f44d7496d090cd0d612de2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731899751035 2024-11-18T03:15:53,985 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeda347f7fb64522b39b618bc1b39062, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731899751708 2024-11-18T03:15:53,985 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting af30865c3b0646edbc4c39ef10ab469d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731899752013 2024-11-18T03:15:53,991 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:53,992 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7c3d8762ece14f5e8ffad3108382bf58 is 50, key is test_row_0/C:col10/1731899752013/Put/seqid=0 2024-11-18T03:15:53,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742118_1294 (size=12207) 2024-11-18T03:15:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899814001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899814002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899814002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899814011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899814011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T03:15:54,066 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/1f4050faec694d1baa97364e8a975761 2024-11-18T03:15:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/d6a69aef3ec5447e988bf473c7c230dc is 50, key is test_row_0/B:col10/1731899752688/Put/seqid=0 2024-11-18T03:15:54,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742119_1295 (size=12051) 2024-11-18T03:15:54,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/d6a69aef3ec5447e988bf473c7c230dc 2024-11-18T03:15:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/5b412efa4c914d5d9ae6b0684c9405e8 is 50, key is test_row_0/C:col10/1731899752688/Put/seqid=0 2024-11-18T03:15:54,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742120_1296 (size=12051) 2024-11-18T03:15:54,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899814211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899814211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899814211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899814220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899814220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,400 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7c3d8762ece14f5e8ffad3108382bf58 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7c3d8762ece14f5e8ffad3108382bf58 2024-11-18T03:15:54,404 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into 7c3d8762ece14f5e8ffad3108382bf58(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:54,404 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:54,404 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899753523; duration=0sec 2024-11-18T03:15:54,404 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:54,404 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:15:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899814521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899814521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899814521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899814531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899814531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:54,545 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/5b412efa4c914d5d9ae6b0684c9405e8 2024-11-18T03:15:54,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/1f4050faec694d1baa97364e8a975761 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1f4050faec694d1baa97364e8a975761 2024-11-18T03:15:54,552 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1f4050faec694d1baa97364e8a975761, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T03:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/d6a69aef3ec5447e988bf473c7c230dc as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/d6a69aef3ec5447e988bf473c7c230dc 2024-11-18T03:15:54,555 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/d6a69aef3ec5447e988bf473c7c230dc, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T03:15:54,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/5b412efa4c914d5d9ae6b0684c9405e8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/5b412efa4c914d5d9ae6b0684c9405e8 2024-11-18T03:15:54,559 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/5b412efa4c914d5d9ae6b0684c9405e8, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T03:15:54,560 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 796efe4cc831df7448a6f85d3ad0d44b in 901ms, sequenceid=132, compaction requested=false 2024-11-18T03:15:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-18T03:15:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-18T03:15:54,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-18T03:15:54,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6670 sec 2024-11-18T03:15:54,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6710 sec 2024-11-18T03:15:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:55,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T03:15:55,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:55,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:15:55,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:55,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:15:55,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:55,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:15:55,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/4e30ad8e8e0643a08876cb43c6f5a77a is 50, key is test_row_0/A:col10/1731899755034/Put/seqid=0 2024-11-18T03:15:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T03:15:55,042 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:55,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-18T03:15:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T03:15:55,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:55,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:55,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742121_1297 (size=16931) 2024-11-18T03:15:55,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899815041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899815041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899815043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899815043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899815047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T03:15:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899815151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899815152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899815152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899815161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:55,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:55,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T03:15:55,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899815361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899815361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899815361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899815371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/4e30ad8e8e0643a08876cb43c6f5a77a 2024-11-18T03:15:55,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/3e50bd9712744c6f870a91e9df680221 is 50, key is test_row_0/B:col10/1731899755034/Put/seqid=0 2024-11-18T03:15:55,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742122_1298 (size=12151) 2024-11-18T03:15:55,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T03:15:55,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899815671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899815671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899815671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899815680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:55,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/3e50bd9712744c6f870a91e9df680221 2024-11-18T03:15:55,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/a53ec05d331844598f96ae46d161208f is 50, key is test_row_0/C:col10/1731899755034/Put/seqid=0 2024-11-18T03:15:55,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742123_1299 (size=12151) 2024-11-18T03:15:55,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:55,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:55,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:55,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:56,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899816060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:56,065 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 796efe4cc831df7448a6f85d3ad0d44b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:56,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:56,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:56,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T03:15:56,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:56,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899816181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899816180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:56,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899816191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899816191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:56,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:56,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:15:56,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/a53ec05d331844598f96ae46d161208f 2024-11-18T03:15:56,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/4e30ad8e8e0643a08876cb43c6f5a77a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/4e30ad8e8e0643a08876cb43c6f5a77a 2024-11-18T03:15:56,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/4e30ad8e8e0643a08876cb43c6f5a77a, entries=250, sequenceid=173, filesize=16.5 K 2024-11-18T03:15:56,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/3e50bd9712744c6f870a91e9df680221 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3e50bd9712744c6f870a91e9df680221 2024-11-18T03:15:56,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3e50bd9712744c6f870a91e9df680221, entries=150, sequenceid=173, filesize=11.9 K 2024-11-18T03:15:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/a53ec05d331844598f96ae46d161208f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a53ec05d331844598f96ae46d161208f 2024-11-18T03:15:56,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a53ec05d331844598f96ae46d161208f, entries=150, sequenceid=173, filesize=11.9 K 2024-11-18T03:15:56,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 796efe4cc831df7448a6f85d3ad0d44b in 1276ms, sequenceid=173, compaction requested=true 2024-11-18T03:15:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:56,311 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:56,311 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:56,311 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41189 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:56,311 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:56,311 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:15:56,311 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:15:56,311 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,311 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,312 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6becb7156309408fb3891b3bdd3dfe00, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1f4050faec694d1baa97364e8a975761, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/4e30ad8e8e0643a08876cb43c6f5a77a] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=40.2 K 2024-11-18T03:15:56,312 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/291bff8b2966410381adae89b4851ec9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/d6a69aef3ec5447e988bf473c7c230dc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3e50bd9712744c6f870a91e9df680221] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=35.6 K 2024-11-18T03:15:56,312 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 291bff8b2966410381adae89b4851ec9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731899752013 2024-11-18T03:15:56,312 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6becb7156309408fb3891b3bdd3dfe00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731899752013 2024-11-18T03:15:56,312 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f4050faec694d1baa97364e8a975761, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731899752675 2024-11-18T03:15:56,313 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a69aef3ec5447e988bf473c7c230dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731899752675 2024-11-18T03:15:56,313 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e30ad8e8e0643a08876cb43c6f5a77a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731899753884 2024-11-18T03:15:56,313 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e50bd9712744c6f870a91e9df680221, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731899753884 2024-11-18T03:15:56,320 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:56,321 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/616042c73a474714814e86408805a9e5 is 50, key is test_row_0/B:col10/1731899755034/Put/seqid=0 2024-11-18T03:15:56,323 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:56,324 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/fd54e11882fd4d94afdca276aae8ee9a is 50, key is test_row_0/A:col10/1731899755034/Put/seqid=0 2024-11-18T03:15:56,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742124_1300 (size=12459) 2024-11-18T03:15:56,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742125_1301 (size=12459) 2024-11-18T03:15:56,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T03:15:56,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,422 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-18T03:15:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:15:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/0ee9d06f60e541adacfc882708400a6c is 50, key is test_row_0/A:col10/1731899755045/Put/seqid=0 2024-11-18T03:15:56,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742126_1302 (size=12151) 2024-11-18T03:15:56,731 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/616042c73a474714814e86408805a9e5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/616042c73a474714814e86408805a9e5 2024-11-18T03:15:56,737 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 616042c73a474714814e86408805a9e5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:56,737 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:56,737 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899756311; duration=0sec 2024-11-18T03:15:56,737 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:56,737 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:15:56,737 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:56,738 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:56,738 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:15:56,738 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:56,739 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7c3d8762ece14f5e8ffad3108382bf58, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/5b412efa4c914d5d9ae6b0684c9405e8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a53ec05d331844598f96ae46d161208f] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=35.6 K 2024-11-18T03:15:56,739 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c3d8762ece14f5e8ffad3108382bf58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731899752013 2024-11-18T03:15:56,739 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b412efa4c914d5d9ae6b0684c9405e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731899752675 2024-11-18T03:15:56,740 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a53ec05d331844598f96ae46d161208f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731899753884 2024-11-18T03:15:56,741 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/fd54e11882fd4d94afdca276aae8ee9a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/fd54e11882fd4d94afdca276aae8ee9a 2024-11-18T03:15:56,746 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into fd54e11882fd4d94afdca276aae8ee9a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:56,746 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:56,746 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899756310; duration=0sec 2024-11-18T03:15:56,746 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:56,746 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:15:56,747 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:56,748 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/24fa5c8fca4245f6a2d0811d1d3654ce is 50, key is test_row_0/C:col10/1731899755034/Put/seqid=0 2024-11-18T03:15:56,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742127_1303 (size=12459) 2024-11-18T03:15:56,758 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/24fa5c8fca4245f6a2d0811d1d3654ce as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/24fa5c8fca4245f6a2d0811d1d3654ce 2024-11-18T03:15:56,763 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into 24fa5c8fca4245f6a2d0811d1d3654ce(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:56,763 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:56,764 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899756311; duration=0sec 2024-11-18T03:15:56,764 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:56,764 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:15:56,840 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/0ee9d06f60e541adacfc882708400a6c 2024-11-18T03:15:56,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/89a4c37b97914da0b85f4c14936d5383 is 50, key is test_row_0/B:col10/1731899755045/Put/seqid=0 2024-11-18T03:15:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742128_1304 (size=12151) 2024-11-18T03:15:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T03:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:57,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:15:57,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899757214, max timestamp=1731899757219Num uniques 859; 2024-11-18T03:15:57,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:57,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899757214, max timestamp=1731899757219Num uniques 859; 2024-11-18T03:15:57,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899757214, max timestamp=1731899757219Num uniques 859; 2024-11-18T03:15:57,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:57,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:15:57,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899817245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731899757238, max timestamp=1731899757248Num uniques 1018; 2024-11-18T03:15:57,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731899757238, max timestamp=1731899757248Num uniques 1018; 2024-11-18T03:15:57,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T03:15:57,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T03:15:57,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1731899757238, max timestamp=1731899757248Num uniques 1018; 2024-11-18T03:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T03:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,255 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/89a4c37b97914da0b85f4c14936d5383 2024-11-18T03:15:57,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/708984eaa5304507a15f491c6ae04850 is 50, key is test_row_0/C:col10/1731899755045/Put/seqid=0 2024-11-18T03:15:57,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742129_1305 (size=12151) 2024-11-18T03:15:57,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731899757270, max timestamp=1731899757318Num uniques 1077; 2024-11-18T03:15:57,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-18T03:15:57,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731899757270, max timestamp=1731899757318Num uniques 1077; 2024-11-18T03:15:57,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731899757270, max timestamp=1731899757318Num uniques 1077; 2024-11-18T03:15:57,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-18T03:15:57,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-18T03:15:57,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899817349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899817350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899817354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899817361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899817461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899817461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899817461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899817571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,665 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/708984eaa5304507a15f491c6ae04850 2024-11-18T03:15:57,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/0ee9d06f60e541adacfc882708400a6c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/0ee9d06f60e541adacfc882708400a6c 2024-11-18T03:15:57,672 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/0ee9d06f60e541adacfc882708400a6c, entries=150, sequenceid=181, filesize=11.9 K 2024-11-18T03:15:57,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899817671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/89a4c37b97914da0b85f4c14936d5383 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/89a4c37b97914da0b85f4c14936d5383 2024-11-18T03:15:57,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899817671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899817671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:57,677 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/89a4c37b97914da0b85f4c14936d5383, entries=150, sequenceid=181, filesize=11.9 K 2024-11-18T03:15:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/708984eaa5304507a15f491c6ae04850 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/708984eaa5304507a15f491c6ae04850 2024-11-18T03:15:57,681 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/708984eaa5304507a15f491c6ae04850, entries=150, sequenceid=181, filesize=11.9 K 2024-11-18T03:15:57,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 796efe4cc831df7448a6f85d3ad0d44b in 1260ms, sequenceid=181, compaction requested=false 2024-11-18T03:15:57,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:57,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:57,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-18T03:15:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-18T03:15:57,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-18T03:15:57,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6390 sec 2024-11-18T03:15:57,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.6460 sec 2024-11-18T03:15:57,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731899757346, max timestamp=1731899757882Num uniques 1136; 2024-11-18T03:15:57,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-18T03:15:57,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731899757346, max timestamp=1731899757882Num uniques 1136; 2024-11-18T03:15:57,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731899757346, max timestamp=1731899757882Num uniques 1136; 2024-11-18T03:15:57,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-18T03:15:57,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-18T03:15:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:15:57,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=53.67 KB heapSize=143.39 KB 2024-11-18T03:15:57,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:57,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/955daa0909f742ec9a82bbd1a7ff6ad6 is 50, key is test_row_0/A:col10/1731899757912/Put/seqid=0 2024-11-18T03:15:57,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742130_1306 (size=12301) 2024-11-18T03:15:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899757983, max timestamp=1731899757992Num uniques 859; 2024-11-18T03:15:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:15:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899757983, max timestamp=1731899757992Num uniques 859; 2024-11-18T03:15:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:15:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899757983, max timestamp=1731899757992Num uniques 859; 2024-11-18T03:15:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:15:58,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899818013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899818015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899818016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899818019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899818081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,085 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 796efe4cc831df7448a6f85d3ad0d44b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:15:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899818121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899818130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899818130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899818131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/955daa0909f742ec9a82bbd1a7ff6ad6 2024-11-18T03:15:58,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899818330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/093cbfe4c26043f0a1a159a4299c7104 is 50, key is test_row_0/B:col10/1731899757912/Put/seqid=0 2024-11-18T03:15:58,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742131_1307 (size=12301) 2024-11-18T03:15:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899818341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899818341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899818341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899818640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899818651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899818651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899818651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:58,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/093cbfe4c26043f0a1a159a4299c7104 2024-11-18T03:15:58,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7dd5705e79a94425b45c47699e6d3ded is 50, key is test_row_0/C:col10/1731899757912/Put/seqid=0 2024-11-18T03:15:58,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742132_1308 (size=12301) 2024-11-18T03:15:59,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7dd5705e79a94425b45c47699e6d3ded 2024-11-18T03:15:59,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899819150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/955daa0909f742ec9a82bbd1a7ff6ad6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/955daa0909f742ec9a82bbd1a7ff6ad6 2024-11-18T03:15:59,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/955daa0909f742ec9a82bbd1a7ff6ad6, entries=150, sequenceid=260, filesize=12.0 K 2024-11-18T03:15:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/093cbfe4c26043f0a1a159a4299c7104 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/093cbfe4c26043f0a1a159a4299c7104 2024-11-18T03:15:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899819161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:59,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/093cbfe4c26043f0a1a159a4299c7104, entries=150, sequenceid=260, filesize=12.0 K 2024-11-18T03:15:59,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7dd5705e79a94425b45c47699e6d3ded as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7dd5705e79a94425b45c47699e6d3ded 2024-11-18T03:15:59,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899819162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:15:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899819162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:15:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:15:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:15:59,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7dd5705e79a94425b45c47699e6d3ded, entries=150, sequenceid=260, filesize=12.0 K 2024-11-18T03:15:59,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~143.34 KB/146784, currentSize=134.18 KB/137400 for 796efe4cc831df7448a6f85d3ad0d44b in 1258ms, sequenceid=260, compaction requested=true 2024-11-18T03:15:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:15:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:15:59,173 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:59,173 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:15:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:59,174 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:59,174 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:59,174 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:15:59,174 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:15:59,174 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:59,174 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:59,174 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/616042c73a474714814e86408805a9e5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/89a4c37b97914da0b85f4c14936d5383, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/093cbfe4c26043f0a1a159a4299c7104] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.0 K 2024-11-18T03:15:59,174 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/fd54e11882fd4d94afdca276aae8ee9a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/0ee9d06f60e541adacfc882708400a6c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/955daa0909f742ec9a82bbd1a7ff6ad6] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.0 K 2024-11-18T03:15:59,174 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 616042c73a474714814e86408805a9e5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731899753884 2024-11-18T03:15:59,174 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd54e11882fd4d94afdca276aae8ee9a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731899753884 2024-11-18T03:15:59,174 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 89a4c37b97914da0b85f4c14936d5383, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731899755036 2024-11-18T03:15:59,175 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee9d06f60e541adacfc882708400a6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731899755036 2024-11-18T03:15:59,175 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 093cbfe4c26043f0a1a159a4299c7104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731899757910 2024-11-18T03:15:59,175 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955daa0909f742ec9a82bbd1a7ff6ad6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731899757910 2024-11-18T03:15:59,182 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:59,182 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:59,182 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/3dfdfa331f51434582db02a329d817fa is 50, key is test_row_0/B:col10/1731899757912/Put/seqid=0 2024-11-18T03:15:59,183 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/93dc549e3cf3449498cc80abae867606 is 50, key is test_row_0/A:col10/1731899757912/Put/seqid=0 2024-11-18T03:15:59,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742133_1309 (size=12711) 2024-11-18T03:15:59,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742134_1310 (size=12711) 2024-11-18T03:15:59,196 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/93dc549e3cf3449498cc80abae867606 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/93dc549e3cf3449498cc80abae867606 2024-11-18T03:15:59,198 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/3dfdfa331f51434582db02a329d817fa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3dfdfa331f51434582db02a329d817fa 2024-11-18T03:15:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T03:15:59,202 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:15:59,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:15:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-18T03:15:59,205 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:15:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T03:15:59,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:15:59,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:15:59,208 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into 93dc549e3cf3449498cc80abae867606(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:59,208 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:59,208 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899759173; duration=0sec 2024-11-18T03:15:59,209 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:15:59,209 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:15:59,209 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:15:59,211 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:15:59,211 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:15:59,211 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:59,211 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 3dfdfa331f51434582db02a329d817fa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:59,211 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/24fa5c8fca4245f6a2d0811d1d3654ce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/708984eaa5304507a15f491c6ae04850, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7dd5705e79a94425b45c47699e6d3ded] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.0 K 2024-11-18T03:15:59,211 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:59,211 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899759173; duration=0sec 2024-11-18T03:15:59,212 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:59,212 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:15:59,212 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24fa5c8fca4245f6a2d0811d1d3654ce, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731899753884 2024-11-18T03:15:59,212 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708984eaa5304507a15f491c6ae04850, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731899755036 2024-11-18T03:15:59,212 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dd5705e79a94425b45c47699e6d3ded, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731899757910 2024-11-18T03:15:59,229 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:15:59,230 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/8d933121e1ef4396a37377e786794c7b is 50, key is test_row_0/C:col10/1731899757912/Put/seqid=0 2024-11-18T03:15:59,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742135_1311 (size=12711) 2024-11-18T03:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T03:15:59,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T03:15:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:15:59,360 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-18T03:15:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:15:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:15:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:15:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:15:59,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/8a843ae111444de0b57086769b26f9ba is 50, key is test_row_0/A:col10/1731899758004/Put/seqid=0 2024-11-18T03:15:59,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742136_1312 (size=12301) 2024-11-18T03:15:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T03:15:59,648 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/8d933121e1ef4396a37377e786794c7b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8d933121e1ef4396a37377e786794c7b 2024-11-18T03:15:59,652 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into 8d933121e1ef4396a37377e786794c7b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:15:59,652 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:15:59,652 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899759173; duration=0sec 2024-11-18T03:15:59,652 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:15:59,652 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:15:59,769 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/8a843ae111444de0b57086769b26f9ba 2024-11-18T03:15:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/1332582544394d80a25924553fcaba21 is 50, key is test_row_0/B:col10/1731899758004/Put/seqid=0 2024-11-18T03:15:59,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742137_1313 (size=12301) 2024-11-18T03:15:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T03:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:00,180 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/1332582544394d80a25924553fcaba21 2024-11-18T03:16:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899820179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899820180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899820181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,186 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in 796efe4cc831df7448a6f85d3ad0d44b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/8e8939b016844f35902422c25825ba0c is 50, key is test_row_0/C:col10/1731899758004/Put/seqid=0 2024-11-18T03:16:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899820185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742138_1314 (size=12301) 2024-11-18T03:16:00,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899820290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899820291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899820301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T03:16:00,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899820501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899820501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899820511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/8e8939b016844f35902422c25825ba0c 2024-11-18T03:16:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/8a843ae111444de0b57086769b26f9ba as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/8a843ae111444de0b57086769b26f9ba 2024-11-18T03:16:00,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/8a843ae111444de0b57086769b26f9ba, entries=150, sequenceid=297, filesize=12.0 K 2024-11-18T03:16:00,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/1332582544394d80a25924553fcaba21 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/1332582544394d80a25924553fcaba21 2024-11-18T03:16:00,605 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/1332582544394d80a25924553fcaba21, entries=150, sequenceid=297, filesize=12.0 K 2024-11-18T03:16:00,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/8e8939b016844f35902422c25825ba0c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8e8939b016844f35902422c25825ba0c 2024-11-18T03:16:00,609 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8e8939b016844f35902422c25825ba0c, entries=150, sequenceid=297, filesize=12.0 K 2024-11-18T03:16:00,610 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 796efe4cc831df7448a6f85d3ad0d44b in 1250ms, sequenceid=297, compaction requested=false 2024-11-18T03:16:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-18T03:16:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-18T03:16:00,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-18T03:16:00,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-18T03:16:00,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4100 sec 2024-11-18T03:16:00,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T03:16:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:00,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:00,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:00,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:00,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:00,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:00,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:00,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/1d09ea84e5e4449896a305ecb25a8f69 is 50, key is test_row_0/A:col10/1731899760812/Put/seqid=0 2024-11-18T03:16:00,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742139_1315 (size=12301) 2024-11-18T03:16:00,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/1d09ea84e5e4449896a305ecb25a8f69 2024-11-18T03:16:00,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/94ce2476c14446de9625c15b83c3cedd is 50, key is test_row_0/B:col10/1731899760812/Put/seqid=0 2024-11-18T03:16:00,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899820856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899820859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899820864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742140_1316 (size=12301) 2024-11-18T03:16:00,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899820970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899820970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899820970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899821188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899821188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899821188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/94ce2476c14446de9625c15b83c3cedd 2024-11-18T03:16:01,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7888dd7b412048b79b2f9eabd3c0a47d is 50, key is test_row_0/C:col10/1731899760812/Put/seqid=0 2024-11-18T03:16:01,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742141_1317 (size=12301) 2024-11-18T03:16:01,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7888dd7b412048b79b2f9eabd3c0a47d 2024-11-18T03:16:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/1d09ea84e5e4449896a305ecb25a8f69 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1d09ea84e5e4449896a305ecb25a8f69 2024-11-18T03:16:01,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1d09ea84e5e4449896a305ecb25a8f69, entries=150, sequenceid=310, filesize=12.0 K 2024-11-18T03:16:01,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/94ce2476c14446de9625c15b83c3cedd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/94ce2476c14446de9625c15b83c3cedd 2024-11-18T03:16:01,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/94ce2476c14446de9625c15b83c3cedd, entries=150, sequenceid=310, filesize=12.0 K 2024-11-18T03:16:01,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/7888dd7b412048b79b2f9eabd3c0a47d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7888dd7b412048b79b2f9eabd3c0a47d 2024-11-18T03:16:01,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7888dd7b412048b79b2f9eabd3c0a47d, entries=150, sequenceid=310, filesize=12.0 K 2024-11-18T03:16:01,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 796efe4cc831df7448a6f85d3ad0d44b in 537ms, sequenceid=310, compaction requested=true 2024-11-18T03:16:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:01,350 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:01,350 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T03:16:01,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:01,351 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:01,351 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:01,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:01,351 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:16:01,351 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:16:01,351 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,351 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,352 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3dfdfa331f51434582db02a329d817fa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/1332582544394d80a25924553fcaba21, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/94ce2476c14446de9625c15b83c3cedd] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.4 K 2024-11-18T03:16:01,352 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/93dc549e3cf3449498cc80abae867606, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/8a843ae111444de0b57086769b26f9ba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1d09ea84e5e4449896a305ecb25a8f69] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.4 K 2024-11-18T03:16:01,352 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93dc549e3cf3449498cc80abae867606, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731899757910 2024-11-18T03:16:01,352 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dfdfa331f51434582db02a329d817fa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731899757910 2024-11-18T03:16:01,352 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1332582544394d80a25924553fcaba21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731899758004 2024-11-18T03:16:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-18T03:16:01,353 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a843ae111444de0b57086769b26f9ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731899758004 2024-11-18T03:16:01,353 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ce2476c14446de9625c15b83c3cedd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899760173 2024-11-18T03:16:01,353 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d09ea84e5e4449896a305ecb25a8f69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899760173 2024-11-18T03:16:01,354 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T03:16:01,355 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:01,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:01,366 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:01,367 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/e8406fef5b064ccd82256adf513d9d45 is 50, key is test_row_0/B:col10/1731899760812/Put/seqid=0 2024-11-18T03:16:01,374 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:01,375 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/d252662f12b44717884d658a0eff07fb is 50, key is test_row_0/A:col10/1731899760812/Put/seqid=0 2024-11-18T03:16:01,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742142_1318 (size=12813) 2024-11-18T03:16:01,403 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/e8406fef5b064ccd82256adf513d9d45 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/e8406fef5b064ccd82256adf513d9d45 2024-11-18T03:16:01,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742143_1319 (size=12813) 2024-11-18T03:16:01,409 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into e8406fef5b064ccd82256adf513d9d45(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:01,409 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:01,409 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899761350; duration=0sec 2024-11-18T03:16:01,410 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:01,410 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:16:01,410 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:01,415 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/d252662f12b44717884d658a0eff07fb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d252662f12b44717884d658a0eff07fb 2024-11-18T03:16:01,421 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into d252662f12b44717884d658a0eff07fb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:01,421 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:01,421 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899761350; duration=0sec 2024-11-18T03:16:01,421 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:01,422 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:16:01,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:01,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:16:01,422 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,422 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8d933121e1ef4396a37377e786794c7b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8e8939b016844f35902422c25825ba0c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7888dd7b412048b79b2f9eabd3c0a47d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.4 K 2024-11-18T03:16:01,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d933121e1ef4396a37377e786794c7b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731899757910 2024-11-18T03:16:01,424 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e8939b016844f35902422c25825ba0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731899758004 2024-11-18T03:16:01,424 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7888dd7b412048b79b2f9eabd3c0a47d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899760173 2024-11-18T03:16:01,478 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#269 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T03:16:01,479 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/f0f2498a91b54d7593582bde77100aee is 50, key is test_row_0/C:col10/1731899760812/Put/seqid=0 2024-11-18T03:16:01,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-18T03:16:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:01,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T03:16:01,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742144_1320 (size=12813) 2024-11-18T03:16:01,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/455843319b7c4a1eaff91b52fecc2a30 is 50, key is test_row_0/A:col10/1731899760844/Put/seqid=0 2024-11-18T03:16:01,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:01,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:01,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:01,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899821551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899821551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899821553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742145_1321 (size=14741) 2024-11-18T03:16:01,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/455843319b7c4a1eaff91b52fecc2a30 2024-11-18T03:16:01,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/394012a787a24f4fb7962767f58ffe4b is 50, key is test_row_0/B:col10/1731899760844/Put/seqid=0 2024-11-18T03:16:01,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742146_1322 (size=12301) 2024-11-18T03:16:01,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/394012a787a24f4fb7962767f58ffe4b 2024-11-18T03:16:01,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/4d1280ebc61e4b83b15aa697a8a83d3f is 50, key is test_row_0/C:col10/1731899760844/Put/seqid=0 2024-11-18T03:16:01,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899821671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899821672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899821672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742147_1323 (size=12301) 2024-11-18T03:16:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T03:16:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:01,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/4d1280ebc61e4b83b15aa697a8a83d3f 2024-11-18T03:16:01,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:01,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:01,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/455843319b7c4a1eaff91b52fecc2a30 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/455843319b7c4a1eaff91b52fecc2a30 2024-11-18T03:16:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T03:16:01,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/455843319b7c4a1eaff91b52fecc2a30, entries=200, sequenceid=333, filesize=14.4 K 2024-11-18T03:16:01,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/394012a787a24f4fb7962767f58ffe4b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/394012a787a24f4fb7962767f58ffe4b 2024-11-18T03:16:01,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/394012a787a24f4fb7962767f58ffe4b, entries=150, sequenceid=333, filesize=12.0 K 2024-11-18T03:16:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/4d1280ebc61e4b83b15aa697a8a83d3f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4d1280ebc61e4b83b15aa697a8a83d3f 2024-11-18T03:16:01,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4d1280ebc61e4b83b15aa697a8a83d3f, entries=150, sequenceid=333, filesize=12.0 K 2024-11-18T03:16:01,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 796efe4cc831df7448a6f85d3ad0d44b in 222ms, sequenceid=333, compaction requested=false 2024-11-18T03:16:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:01,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T03:16:01,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:01,841 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T03:16:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/53ce575aec7d47b9832c1913f958759c is 50, key is test_row_0/A:col10/1731899761546/Put/seqid=0 2024-11-18T03:16:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:01,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:01,930 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/f0f2498a91b54d7593582bde77100aee as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f0f2498a91b54d7593582bde77100aee 2024-11-18T03:16:01,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742148_1324 (size=12301) 2024-11-18T03:16:01,942 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into f0f2498a91b54d7593582bde77100aee(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:01,943 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:01,943 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899761350; duration=0sec 2024-11-18T03:16:01,943 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:01,943 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:16:01,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899761916, max timestamp=1731899761941Num uniques 859; 2024-11-18T03:16:01,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:01,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899821969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899821971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899821974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T03:16:02,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899822092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899822092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899822092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47064 deadline: 1731899822122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,127 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 796efe4cc831df7448a6f85d3ad0d44b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899822201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,206 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in 796efe4cc831df7448a6f85d3ad0d44b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899822301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899822301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899822311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,331 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/53ce575aec7d47b9832c1913f958759c 2024-11-18T03:16:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/40a5c4685c484ca0ad0fe2d66e76f888 is 50, key is test_row_0/B:col10/1731899761546/Put/seqid=0 2024-11-18T03:16:02,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742149_1325 (size=12301) 2024-11-18T03:16:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T03:16:02,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899822611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899822612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899822621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:02,780 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/40a5c4685c484ca0ad0fe2d66e76f888 2024-11-18T03:16:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/22b0081927c34acb811940e287649691 is 50, key is test_row_0/C:col10/1731899761546/Put/seqid=0 2024-11-18T03:16:02,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742150_1326 (size=12301) 2024-11-18T03:16:02,844 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/22b0081927c34acb811940e287649691 2024-11-18T03:16:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/53ce575aec7d47b9832c1913f958759c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/53ce575aec7d47b9832c1913f958759c 2024-11-18T03:16:02,859 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/53ce575aec7d47b9832c1913f958759c, entries=150, sequenceid=344, filesize=12.0 K 2024-11-18T03:16:02,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/40a5c4685c484ca0ad0fe2d66e76f888 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/40a5c4685c484ca0ad0fe2d66e76f888 2024-11-18T03:16:02,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/40a5c4685c484ca0ad0fe2d66e76f888, entries=150, sequenceid=344, filesize=12.0 K 2024-11-18T03:16:02,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/22b0081927c34acb811940e287649691 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/22b0081927c34acb811940e287649691 2024-11-18T03:16:02,898 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/22b0081927c34acb811940e287649691, entries=150, sequenceid=344, filesize=12.0 K 2024-11-18T03:16:02,899 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 796efe4cc831df7448a6f85d3ad0d44b in 1058ms, sequenceid=344, compaction requested=true 2024-11-18T03:16:02,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:02,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:02,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-18T03:16:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-18T03:16:02,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-18T03:16:02,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5460 sec 2024-11-18T03:16:02,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5530 sec 2024-11-18T03:16:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:03,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-11-18T03:16:03,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:03,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:03,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:03,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:03,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:03,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:03,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/754c43962677418db9457ce751b37282 is 50, key is test_row_0/A:col10/1731899761958/Put/seqid=0 2024-11-18T03:16:03,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742151_1327 (size=12301) 2024-11-18T03:16:03,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/754c43962677418db9457ce751b37282 2024-11-18T03:16:03,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899823173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/34a62f98f72a469fa163307e95746b75 is 50, key is test_row_0/B:col10/1731899761958/Put/seqid=0 2024-11-18T03:16:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899823178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899823177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742152_1328 (size=12301) 2024-11-18T03:16:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899823291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899823291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899823291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899823501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899823511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899823511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T03:16:03,532 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:03,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-18T03:16:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T03:16:03,537 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:03,538 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:03,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:03,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/34a62f98f72a469fa163307e95746b75 2024-11-18T03:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T03:16:03,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/49828163576846219100081bb2a37792 is 50, key is test_row_0/C:col10/1731899761958/Put/seqid=0 2024-11-18T03:16:03,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742153_1329 (size=12301) 2024-11-18T03:16:03,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T03:16:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:03,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899823811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899823822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899823822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:03,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T03:16:03,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:03,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:03,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:03,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T03:16:03,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T03:16:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:03,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:04,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/49828163576846219100081bb2a37792 2024-11-18T03:16:04,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/754c43962677418db9457ce751b37282 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/754c43962677418db9457ce751b37282 2024-11-18T03:16:04,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/754c43962677418db9457ce751b37282, entries=150, sequenceid=372, filesize=12.0 K 2024-11-18T03:16:04,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/34a62f98f72a469fa163307e95746b75 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/34a62f98f72a469fa163307e95746b75 2024-11-18T03:16:04,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/34a62f98f72a469fa163307e95746b75, entries=150, sequenceid=372, filesize=12.0 K 2024-11-18T03:16:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/49828163576846219100081bb2a37792 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/49828163576846219100081bb2a37792 2024-11-18T03:16:04,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/49828163576846219100081bb2a37792, entries=150, sequenceid=372, filesize=12.0 K 2024-11-18T03:16:04,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for 796efe4cc831df7448a6f85d3ad0d44b in 946ms, sequenceid=372, compaction requested=true 2024-11-18T03:16:04,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:04,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:04,087 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:16:04,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:04,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:04,088 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:04,089 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:16:04,089 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:16:04,089 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:16:04,089 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:16:04,089 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:04,089 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:04,089 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/e8406fef5b064ccd82256adf513d9d45, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/394012a787a24f4fb7962767f58ffe4b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/40a5c4685c484ca0ad0fe2d66e76f888, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/34a62f98f72a469fa163307e95746b75] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=48.6 K 2024-11-18T03:16:04,089 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d252662f12b44717884d658a0eff07fb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/455843319b7c4a1eaff91b52fecc2a30, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/53ce575aec7d47b9832c1913f958759c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/754c43962677418db9457ce751b37282] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=50.9 K 2024-11-18T03:16:04,090 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d252662f12b44717884d658a0eff07fb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899760173 2024-11-18T03:16:04,091 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e8406fef5b064ccd82256adf513d9d45, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899760173 2024-11-18T03:16:04,091 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 394012a787a24f4fb7962767f58ffe4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731899760844 2024-11-18T03:16:04,091 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 455843319b7c4a1eaff91b52fecc2a30, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731899760844 2024-11-18T03:16:04,092 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a5c4685c484ca0ad0fe2d66e76f888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731899761540 2024-11-18T03:16:04,092 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53ce575aec7d47b9832c1913f958759c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731899761540 2024-11-18T03:16:04,092 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 754c43962677418db9457ce751b37282, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731899761958 2024-11-18T03:16:04,092 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 34a62f98f72a469fa163307e95746b75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731899761958 2024-11-18T03:16:04,103 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:04,103 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#279 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:04,104 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/19471183363e42de991a180e57aefe1c is 50, key is test_row_0/A:col10/1731899761958/Put/seqid=0 2024-11-18T03:16:04,104 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/8275a1e3c230444aab57bdf86f71be14 is 50, key is test_row_0/B:col10/1731899761958/Put/seqid=0 2024-11-18T03:16:04,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742154_1330 (size=12949) 2024-11-18T03:16:04,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742155_1331 (size=12949) 2024-11-18T03:16:04,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T03:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:04,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:04,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:04,158 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/19471183363e42de991a180e57aefe1c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/19471183363e42de991a180e57aefe1c 2024-11-18T03:16:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T03:16:04,166 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into 19471183363e42de991a180e57aefe1c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:04,166 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:04,166 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=12, startTime=1731899764087; duration=0sec 2024-11-18T03:16:04,166 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:04,166 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:16:04,166 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:16:04,170 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:16:04,170 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:16:04,170 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:04,170 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f0f2498a91b54d7593582bde77100aee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4d1280ebc61e4b83b15aa697a8a83d3f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/22b0081927c34acb811940e287649691, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/49828163576846219100081bb2a37792] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=48.6 K 2024-11-18T03:16:04,170 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f2498a91b54d7593582bde77100aee, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731899760173 2024-11-18T03:16:04,171 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d1280ebc61e4b83b15aa697a8a83d3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731899760844 2024-11-18T03:16:04,171 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b0081927c34acb811940e287649691, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731899761540 2024-11-18T03:16:04,172 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49828163576846219100081bb2a37792, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731899761958 2024-11-18T03:16:04,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/a29f8bccbbfe468d872595ae8de9783d is 50, key is test_row_0/A:col10/1731899763173/Put/seqid=0 2024-11-18T03:16:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742156_1332 (size=12301) 2024-11-18T03:16:04,203 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/a29f8bccbbfe468d872595ae8de9783d 2024-11-18T03:16:04,223 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:04,224 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/e21204c13fa74c1b9cf6ab4614cbaa15 is 50, key is test_row_0/C:col10/1731899761958/Put/seqid=0 2024-11-18T03:16:04,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/faa6664351314042b2290832c8d6bde0 is 50, key is test_row_0/B:col10/1731899763173/Put/seqid=0 2024-11-18T03:16:04,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742158_1334 (size=12301) 2024-11-18T03:16:04,268 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/faa6664351314042b2290832c8d6bde0 2024-11-18T03:16:04,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742157_1333 (size=12949) 2024-11-18T03:16:04,285 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/e21204c13fa74c1b9cf6ab4614cbaa15 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/e21204c13fa74c1b9cf6ab4614cbaa15 2024-11-18T03:16:04,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/b560a62f7c524562b4f6bf96c0f52afa is 50, key is test_row_0/C:col10/1731899763173/Put/seqid=0 2024-11-18T03:16:04,297 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into e21204c13fa74c1b9cf6ab4614cbaa15(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:04,297 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:04,297 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=12, startTime=1731899764088; duration=0sec 2024-11-18T03:16:04,297 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:04,297 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:16:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:04,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742159_1335 (size=12301) 2024-11-18T03:16:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899764380, max timestamp=1731899764399Num uniques 859; 2024-11-18T03:16:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:04,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899764380, max timestamp=1731899764399Num uniques 859; 2024-11-18T03:16:04,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:04,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899764380, max timestamp=1731899764399Num uniques 859; 2024-11-18T03:16:04,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:04,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731899764463, max timestamp=1731899764488Num uniques 1018; 2024-11-18T03:16:04,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T03:16:04,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731899764463, max timestamp=1731899764488Num uniques 1018; 2024-11-18T03:16:04,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T03:16:04,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731899764463, max timestamp=1731899764488Num uniques 1018; 2024-11-18T03:16:04,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T03:16:04,553 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/8275a1e3c230444aab57bdf86f71be14 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/8275a1e3c230444aab57bdf86f71be14 2024-11-18T03:16:04,585 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 8275a1e3c230444aab57bdf86f71be14(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:04,585 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:04,585 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=12, startTime=1731899764087; duration=0sec 2024-11-18T03:16:04,585 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:04,585 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:16:04,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899824585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899824587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899824597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T03:16:04,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899824701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899824704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899824711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,743 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/b560a62f7c524562b4f6bf96c0f52afa 2024-11-18T03:16:04,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/a29f8bccbbfe468d872595ae8de9783d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/a29f8bccbbfe468d872595ae8de9783d 2024-11-18T03:16:04,751 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/a29f8bccbbfe468d872595ae8de9783d, entries=150, sequenceid=382, filesize=12.0 K 2024-11-18T03:16:04,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/faa6664351314042b2290832c8d6bde0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faa6664351314042b2290832c8d6bde0 2024-11-18T03:16:04,757 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faa6664351314042b2290832c8d6bde0, entries=150, sequenceid=382, filesize=12.0 K 2024-11-18T03:16:04,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/b560a62f7c524562b4f6bf96c0f52afa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/b560a62f7c524562b4f6bf96c0f52afa 2024-11-18T03:16:04,763 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/b560a62f7c524562b4f6bf96c0f52afa, entries=150, sequenceid=382, filesize=12.0 K 2024-11-18T03:16:04,764 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 796efe4cc831df7448a6f85d3ad0d44b in 609ms, sequenceid=382, compaction requested=false 2024-11-18T03:16:04,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:04,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:04,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-18T03:16:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-18T03:16:04,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-18T03:16:04,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2280 sec 2024-11-18T03:16:04,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.2370 sec 2024-11-18T03:16:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:04,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=147.60 KB heapSize=424.69 KB 2024-11-18T03:16:04,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731899764531, max timestamp=1731899764912Num uniques 1027; 2024-11-18T03:16:04,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899764531, max timestamp=1731899764914Num uniques 1027; 2024-11-18T03:16:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:16:04,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-18T03:16:04,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47026, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-18T03:16:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:04,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/73c22020ddcf46df80701886a27981d2 is 50, key is test_row_0/A:col10/1731899764914/Put/seqid=0 2024-11-18T03:16:04,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742160_1336 (size=12301) 2024-11-18T03:16:04,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/73c22020ddcf46df80701886a27981d2 2024-11-18T03:16:04,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/cf8369524bdc48868a56f937cead5056 is 50, key is test_row_0/B:col10/1731899764914/Put/seqid=0 2024-11-18T03:16:04,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742161_1337 (size=12301) 2024-11-18T03:16:04,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/cf8369524bdc48868a56f937cead5056 2024-11-18T03:16:04,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899824961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899824961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899824967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:04,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/3876ce2d7ff0437e881f6886e8115fdb is 50, key is test_row_0/C:col10/1731899764914/Put/seqid=0 2024-11-18T03:16:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742162_1338 (size=12301) 2024-11-18T03:16:05,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899825071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899825071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:05,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899825081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:05,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:05,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899825281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:05,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899825281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:05,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899825291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:05,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.98 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/3876ce2d7ff0437e881f6886e8115fdb 2024-11-18T03:16:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/73c22020ddcf46df80701886a27981d2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/73c22020ddcf46df80701886a27981d2 2024-11-18T03:16:05,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/73c22020ddcf46df80701886a27981d2, entries=150, sequenceid=441, filesize=12.0 K 2024-11-18T03:16:05,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/cf8369524bdc48868a56f937cead5056 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/cf8369524bdc48868a56f937cead5056 2024-11-18T03:16:05,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/cf8369524bdc48868a56f937cead5056, entries=150, sequenceid=441, filesize=12.0 K 2024-11-18T03:16:05,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/3876ce2d7ff0437e881f6886e8115fdb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/3876ce2d7ff0437e881f6886e8115fdb 2024-11-18T03:16:05,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/3876ce2d7ff0437e881f6886e8115fdb, entries=150, sequenceid=441, filesize=12.0 K 2024-11-18T03:16:05,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~70.37 KB/72054, heapSize ~195.09 KB/199768, currentSize=114.05 KB/116790 for 796efe4cc831df7448a6f85d3ad0d44b in 530ms, sequenceid=441, compaction requested=true 2024-11-18T03:16:05,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:05,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:05,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:05,447 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:05,447 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:05,448 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:05,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:05,448 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:16:05,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:05,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:05,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:05,448 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:05,448 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/19471183363e42de991a180e57aefe1c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/a29f8bccbbfe468d872595ae8de9783d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/73c22020ddcf46df80701886a27981d2] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.7 K 2024-11-18T03:16:05,449 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:05,449 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19471183363e42de991a180e57aefe1c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731899761958 2024-11-18T03:16:05,449 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:16:05,449 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:05,449 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/8275a1e3c230444aab57bdf86f71be14, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faa6664351314042b2290832c8d6bde0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/cf8369524bdc48868a56f937cead5056] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.7 K 2024-11-18T03:16:05,454 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a29f8bccbbfe468d872595ae8de9783d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731899763155 2024-11-18T03:16:05,455 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8275a1e3c230444aab57bdf86f71be14, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731899761958 2024-11-18T03:16:05,455 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c22020ddcf46df80701886a27981d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731899764531 2024-11-18T03:16:05,458 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting faa6664351314042b2290832c8d6bde0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731899763155 2024-11-18T03:16:05,459 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8369524bdc48868a56f937cead5056, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731899764531 2024-11-18T03:16:05,487 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:05,487 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/742514c894aa47939843ec32e8c70d74 is 50, key is test_row_0/B:col10/1731899764914/Put/seqid=0 2024-11-18T03:16:05,494 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:05,495 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/6ba4beece2dd400395e83ef81b2e2361 is 50, key is test_row_0/A:col10/1731899764914/Put/seqid=0 2024-11-18T03:16:05,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742163_1339 (size=13051) 2024-11-18T03:16:05,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742164_1340 (size=13051) 2024-11-18T03:16:05,516 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/742514c894aa47939843ec32e8c70d74 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/742514c894aa47939843ec32e8c70d74 2024-11-18T03:16:05,524 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 742514c894aa47939843ec32e8c70d74(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:05,524 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:05,538 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899765447; duration=0sec 2024-11-18T03:16:05,539 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:05,552 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:16:05,552 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:05,555 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:05,555 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:16:05,555 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:05,555 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/e21204c13fa74c1b9cf6ab4614cbaa15, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/b560a62f7c524562b4f6bf96c0f52afa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/3876ce2d7ff0437e881f6886e8115fdb] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.7 K 2024-11-18T03:16:05,556 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e21204c13fa74c1b9cf6ab4614cbaa15, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731899761958 2024-11-18T03:16:05,556 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b560a62f7c524562b4f6bf96c0f52afa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731899763155 2024-11-18T03:16:05,558 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3876ce2d7ff0437e881f6886e8115fdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731899764531 2024-11-18T03:16:05,582 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:05,583 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/4b5365db823c4a9f8dbef3992223bdbf is 50, key is test_row_0/C:col10/1731899764914/Put/seqid=0 2024-11-18T03:16:05,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899764951, max timestamp=1731899765592Num uniques 859; 2024-11-18T03:16:05,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:05,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899764951, max timestamp=1731899765595Num uniques 859; 2024-11-18T03:16:05,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899764951, max timestamp=1731899765595Num uniques 859; 2024-11-18T03:16:05,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:05,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:05,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-18T03:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:05,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/7046cb9903854cdbb9c4d51b121576cb is 50, key is test_row_0/A:col10/1731899764959/Put/seqid=0 2024-11-18T03:16:05,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742165_1341 (size=13051) 2024-11-18T03:16:05,612 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/4b5365db823c4a9f8dbef3992223bdbf as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4b5365db823c4a9f8dbef3992223bdbf 2024-11-18T03:16:05,618 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into 4b5365db823c4a9f8dbef3992223bdbf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:05,618 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:05,618 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899765448; duration=0sec 2024-11-18T03:16:05,619 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:05,619 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:16:05,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742166_1342 (size=12301) 2024-11-18T03:16:05,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899765626, max timestamp=1731899765644Num uniques 859; 2024-11-18T03:16:05,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:16:05,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899765631, max timestamp=1731899765644Num uniques 859; 2024-11-18T03:16:05,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731899765631, max timestamp=1731899765644Num uniques 859; 2024-11-18T03:16:05,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:16:05,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T03:16:05,681 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:05,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-18T03:16:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T03:16:05,683 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:05,684 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:05,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:05,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899765667, max timestamp=1731899765702Num uniques 1068; 2024-11-18T03:16:05,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-18T03:16:05,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899765667, max timestamp=1731899765702Num uniques 1118; 2024-11-18T03:16:05,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-11-18T03:16:05,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899765667, max timestamp=1731899765702Num uniques 1068; 2024-11-18T03:16:05,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-18T03:16:05,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731899765754, max timestamp=1731899765764Num uniques 1009; 2024-11-18T03:16:05,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731899765754, max timestamp=1731899765764Num uniques 1009; 2024-11-18T03:16:05,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-18T03:16:05,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731899765754, max timestamp=1731899765764Num uniques 1009; 2024-11-18T03:16:05,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-18T03:16:05,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-18T03:16:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T03:16:05,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731899765801, max timestamp=1731899765811Num uniques 1086; 2024-11-18T03:16:05,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T03:16:05,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731899765801, max timestamp=1731899765811Num uniques 1086; 2024-11-18T03:16:05,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T03:16:05,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731899765801, max timestamp=1731899765811Num uniques 1086; 2024-11-18T03:16:05,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T03:16:05,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T03:16:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:05,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:05,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:05,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731899765842, max timestamp=1731899765862Num uniques 1045; 2024-11-18T03:16:05,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T03:16:05,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731899765842, max timestamp=1731899765862Num uniques 1045; 2024-11-18T03:16:05,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T03:16:05,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731899765842, max timestamp=1731899765862Num uniques 1045; 2024-11-18T03:16:05,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T03:16:05,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731899765903, max timestamp=1731899765917Num uniques 1104; 2024-11-18T03:16:05,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-18T03:16:05,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731899765903, max timestamp=1731899765917Num uniques 1104; 2024-11-18T03:16:05,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-18T03:16:05,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731899765903, max timestamp=1731899765917Num uniques 1104; 2024-11-18T03:16:05,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-18T03:16:05,927 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/6ba4beece2dd400395e83ef81b2e2361 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6ba4beece2dd400395e83ef81b2e2361 2024-11-18T03:16:05,943 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into 6ba4beece2dd400395e83ef81b2e2361(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:05,943 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:05,943 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899765446; duration=0sec 2024-11-18T03:16:05,943 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:05,943 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:16:05,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T03:16:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:05,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:05,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899825982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:05,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731899765966, max timestamp=1731899765990Num uniques 1113; 2024-11-18T03:16:05,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-18T03:16:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:05,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731899765966, max timestamp=1731899765990Num uniques 1113; 2024-11-18T03:16:05,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-18T03:16:06,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731899765966, max timestamp=1731899765990Num uniques 1113; 2024-11-18T03:16:06,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-18T03:16:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T03:16:06,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/7046cb9903854cdbb9c4d51b121576cb 2024-11-18T03:16:06,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/66a15c03d6274a0c9baade7ac2e59e32 is 50, key is test_row_0/B:col10/1731899764959/Put/seqid=0 2024-11-18T03:16:06,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742167_1343 (size=12301) 2024-11-18T03:16:06,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/66a15c03d6274a0c9baade7ac2e59e32 2024-11-18T03:16:06,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731899766052, max timestamp=1731899766095Num uniques 1022; 2024-11-18T03:16:06,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731899766052, max timestamp=1731899766095Num uniques 1022; 2024-11-18T03:16:06,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-18T03:16:06,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-18T03:16:06,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731899766052, max timestamp=1731899766095Num uniques 1022; 2024-11-18T03:16:06,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-18T03:16:06,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/05d370253b3842d9940df2d1a929651a is 50, key is test_row_0/C:col10/1731899764959/Put/seqid=0 2024-11-18T03:16:06,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742168_1344 (size=12301) 2024-11-18T03:16:06,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/05d370253b3842d9940df2d1a929651a 2024-11-18T03:16:06,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T03:16:06,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/7046cb9903854cdbb9c4d51b121576cb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7046cb9903854cdbb9c4d51b121576cb 2024-11-18T03:16:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:06,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7046cb9903854cdbb9c4d51b121576cb, entries=150, sequenceid=464, filesize=12.0 K 2024-11-18T03:16:06,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/66a15c03d6274a0c9baade7ac2e59e32 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/66a15c03d6274a0c9baade7ac2e59e32 2024-11-18T03:16:06,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/66a15c03d6274a0c9baade7ac2e59e32, entries=150, sequenceid=464, filesize=12.0 K 2024-11-18T03:16:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/05d370253b3842d9940df2d1a929651a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/05d370253b3842d9940df2d1a929651a 2024-11-18T03:16:06,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/05d370253b3842d9940df2d1a929651a, entries=150, sequenceid=464, filesize=12.0 K 2024-11-18T03:16:06,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=147.60 KB/151140 for 796efe4cc831df7448a6f85d3ad0d44b in 592ms, sequenceid=464, compaction requested=false 2024-11-18T03:16:06,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:06,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=154.31 KB heapSize=434.86 KB 2024-11-18T03:16:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:16:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:16:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T03:16:06,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/5ee2492fa5c142d7a124b98ec5b5f51d is 50, key is test_row_0/A:col10/1731899766159/Put/seqid=0 2024-11-18T03:16:06,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899826208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899826209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899826214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742169_1345 (size=19621) 2024-11-18T03:16:06,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/5ee2492fa5c142d7a124b98ec5b5f51d 2024-11-18T03:16:06,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47066 deadline: 1731899826256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,262 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in 796efe4cc831df7448a6f85d3ad0d44b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/0723e06bb5f341b4a2323c5d66f156f8 is 50, key is test_row_0/B:col10/1731899766159/Put/seqid=0 2024-11-18T03:16:06,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T03:16:06,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:06,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:06,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T03:16:06,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742170_1346 (size=12301) 2024-11-18T03:16:06,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/0723e06bb5f341b4a2323c5d66f156f8 2024-11-18T03:16:06,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899826322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899826322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899826322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d9f95ee2d40b41e29172859d18680a7b is 50, key is test_row_0/C:col10/1731899766159/Put/seqid=0 2024-11-18T03:16:06,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742171_1347 (size=12301) 2024-11-18T03:16:06,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d9f95ee2d40b41e29172859d18680a7b 2024-11-18T03:16:06,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/5ee2492fa5c142d7a124b98ec5b5f51d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/5ee2492fa5c142d7a124b98ec5b5f51d 2024-11-18T03:16:06,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/5ee2492fa5c142d7a124b98ec5b5f51d, entries=300, sequenceid=624, filesize=19.2 K 2024-11-18T03:16:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/0723e06bb5f341b4a2323c5d66f156f8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/0723e06bb5f341b4a2323c5d66f156f8 2024-11-18T03:16:06,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/0723e06bb5f341b4a2323c5d66f156f8, entries=150, sequenceid=624, filesize=12.0 K 2024-11-18T03:16:06,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d9f95ee2d40b41e29172859d18680a7b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d9f95ee2d40b41e29172859d18680a7b 2024-11-18T03:16:06,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d9f95ee2d40b41e29172859d18680a7b, entries=150, sequenceid=624, filesize=12.0 K 2024-11-18T03:16:06,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.81 KB/445248, currentSize=26.84 KB/27480 for 796efe4cc831df7448a6f85d3ad0d44b in 230ms, sequenceid=624, compaction requested=true 2024-11-18T03:16:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:06,422 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:16:06,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:06,423 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:06,423 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:06,423 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:16:06,423 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:16:06,423 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,423 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,424 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/742514c894aa47939843ec32e8c70d74, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/66a15c03d6274a0c9baade7ac2e59e32, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/0723e06bb5f341b4a2323c5d66f156f8] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.8 K 2024-11-18T03:16:06,424 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6ba4beece2dd400395e83ef81b2e2361, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7046cb9903854cdbb9c4d51b121576cb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/5ee2492fa5c142d7a124b98ec5b5f51d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=43.9 K 2024-11-18T03:16:06,424 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba4beece2dd400395e83ef81b2e2361, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731899764531 2024-11-18T03:16:06,424 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 742514c894aa47939843ec32e8c70d74, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731899764531 2024-11-18T03:16:06,424 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7046cb9903854cdbb9c4d51b121576cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731899764959 2024-11-18T03:16:06,424 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a15c03d6274a0c9baade7ac2e59e32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731899764959 2024-11-18T03:16:06,425 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0723e06bb5f341b4a2323c5d66f156f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1731899766159 2024-11-18T03:16:06,425 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee2492fa5c142d7a124b98ec5b5f51d, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1731899766147 2024-11-18T03:16:06,438 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:06,438 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/47d6ee900f8c46ceafb74d186b00daaf is 50, key is test_row_0/A:col10/1731899766159/Put/seqid=0 2024-11-18T03:16:06,441 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:06,441 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/2ca6d7ad5e4544d0b0ef0e2d606f5fee is 50, key is test_row_0/B:col10/1731899766159/Put/seqid=0 2024-11-18T03:16:06,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T03:16:06,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742172_1348 (size=13153) 2024-11-18T03:16:06,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,465 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T03:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:06,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/6f5cddf4cd07483c850e544e53a6581f is 50, key is test_row_0/A:col10/1731899766211/Put/seqid=0 2024-11-18T03:16:06,481 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/47d6ee900f8c46ceafb74d186b00daaf as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/47d6ee900f8c46ceafb74d186b00daaf 2024-11-18T03:16:06,487 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into 47d6ee900f8c46ceafb74d186b00daaf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:06,487 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:06,488 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899766422; duration=0sec 2024-11-18T03:16:06,488 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:06,488 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:16:06,488 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:06,489 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:06,489 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:16:06,490 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:06,490 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4b5365db823c4a9f8dbef3992223bdbf, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/05d370253b3842d9940df2d1a929651a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d9f95ee2d40b41e29172859d18680a7b] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.8 K 2024-11-18T03:16:06,490 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b5365db823c4a9f8dbef3992223bdbf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731899764531 2024-11-18T03:16:06,491 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d370253b3842d9940df2d1a929651a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731899764959 2024-11-18T03:16:06,491 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9f95ee2d40b41e29172859d18680a7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1731899766159 2024-11-18T03:16:06,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742173_1349 (size=13153) 2024-11-18T03:16:06,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742174_1350 (size=12301) 2024-11-18T03:16:06,503 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/6f5cddf4cd07483c850e544e53a6581f 2024-11-18T03:16:06,517 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:06,517 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d96dbb9b12984080b969f9cdd7f33a33 is 50, key is test_row_0/C:col10/1731899766159/Put/seqid=0 2024-11-18T03:16:06,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/17ab3e3d1a5048e6bbaec1d7188a1b73 is 50, key is test_row_0/B:col10/1731899766211/Put/seqid=0 2024-11-18T03:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:06,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:06,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742175_1351 (size=13153) 2024-11-18T03:16:06,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742176_1352 (size=12301) 2024-11-18T03:16:06,573 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/17ab3e3d1a5048e6bbaec1d7188a1b73 2024-11-18T03:16:06,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/a7cfda5c60384956942df4119787a41d is 50, key is test_row_0/C:col10/1731899766211/Put/seqid=0 2024-11-18T03:16:06,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742177_1353 (size=12301) 2024-11-18T03:16:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899766581, max timestamp=1731899766629Num uniques 859; 2024-11-18T03:16:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899766581, max timestamp=1731899766629Num uniques 859; 2024-11-18T03:16:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:06,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899826678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899826680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899826680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899826791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899826800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899826800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T03:16:06,907 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/2ca6d7ad5e4544d0b0ef0e2d606f5fee as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/2ca6d7ad5e4544d0b0ef0e2d606f5fee 2024-11-18T03:16:06,913 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 2ca6d7ad5e4544d0b0ef0e2d606f5fee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:06,913 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:06,913 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899766422; duration=0sec 2024-11-18T03:16:06,913 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:06,913 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:16:06,981 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d96dbb9b12984080b969f9cdd7f33a33 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d96dbb9b12984080b969f9cdd7f33a33 2024-11-18T03:16:07,009 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into d96dbb9b12984080b969f9cdd7f33a33(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:07,009 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:07,009 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899766422; duration=0sec 2024-11-18T03:16:07,009 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:07,009 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:16:07,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899827002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899827012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,024 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/a7cfda5c60384956942df4119787a41d 2024-11-18T03:16:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899827015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/6f5cddf4cd07483c850e544e53a6581f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6f5cddf4cd07483c850e544e53a6581f 2024-11-18T03:16:07,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6f5cddf4cd07483c850e544e53a6581f, entries=150, sequenceid=631, filesize=12.0 K 2024-11-18T03:16:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/17ab3e3d1a5048e6bbaec1d7188a1b73 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/17ab3e3d1a5048e6bbaec1d7188a1b73 2024-11-18T03:16:07,051 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/17ab3e3d1a5048e6bbaec1d7188a1b73, entries=150, sequenceid=631, filesize=12.0 K 2024-11-18T03:16:07,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/a7cfda5c60384956942df4119787a41d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a7cfda5c60384956942df4119787a41d 2024-11-18T03:16:07,059 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a7cfda5c60384956942df4119787a41d, entries=150, sequenceid=631, filesize=12.0 K 2024-11-18T03:16:07,059 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=158.78 KB/162590 for 796efe4cc831df7448a6f85d3ad0d44b in 594ms, sequenceid=631, compaction requested=false 2024-11-18T03:16:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-18T03:16:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-18T03:16:07,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-18T03:16:07,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3780 sec 2024-11-18T03:16:07,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.3840 sec 2024-11-18T03:16:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:07,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=165.49 KB heapSize=482.92 KB 2024-11-18T03:16:07,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899827337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899827342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899827346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/032cd025481c464d9bb3ca619857846d is 50, key is test_row_0/A:col10/1731899766678/Put/seqid=0 2024-11-18T03:16:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742178_1354 (size=12301) 2024-11-18T03:16:07,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899827451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899827451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899827461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899827661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899827661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899827672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/032cd025481c464d9bb3ca619857846d 2024-11-18T03:16:07,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/b8770333320045ae8d82df6e46e4cc2c is 50, key is test_row_0/B:col10/1731899766678/Put/seqid=0 2024-11-18T03:16:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742179_1355 (size=12301) 2024-11-18T03:16:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T03:16:07,841 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:07,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-18T03:16:07,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T03:16:07,846 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:07,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T03:16:07,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899827971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899827972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:07,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899827981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:08,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T03:16:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T03:16:08,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T03:16:08,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:08,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/b8770333320045ae8d82df6e46e4cc2c 2024-11-18T03:16:08,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/bdcda01b57374ef68840338347f8d98e is 50, key is test_row_0/C:col10/1731899766678/Put/seqid=0 2024-11-18T03:16:08,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742180_1356 (size=12301) 2024-11-18T03:16:08,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T03:16:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T03:16:08,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T03:16:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47038 deadline: 1731899828481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:08,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47048 deadline: 1731899828481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47040 deadline: 1731899828491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:08,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T03:16:08,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. as already flushing 2024-11-18T03:16:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:08,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76.04 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/bdcda01b57374ef68840338347f8d98e 2024-11-18T03:16:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/032cd025481c464d9bb3ca619857846d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/032cd025481c464d9bb3ca619857846d 2024-11-18T03:16:08,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/032cd025481c464d9bb3ca619857846d, entries=150, sequenceid=671, filesize=12.0 K 2024-11-18T03:16:08,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/b8770333320045ae8d82df6e46e4cc2c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/b8770333320045ae8d82df6e46e4cc2c 2024-11-18T03:16:08,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/b8770333320045ae8d82df6e46e4cc2c, entries=150, sequenceid=671, filesize=12.0 K 2024-11-18T03:16:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/bdcda01b57374ef68840338347f8d98e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/bdcda01b57374ef68840338347f8d98e 2024-11-18T03:16:08,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/bdcda01b57374ef68840338347f8d98e, entries=150, sequenceid=671, filesize=12.0 K 2024-11-18T03:16:08,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~165.49 KB/169460, heapSize ~482.88 KB/494464, currentSize=20.13 KB/20610 for 796efe4cc831df7448a6f85d3ad0d44b in 1356ms, sequenceid=671, compaction requested=true 2024-11-18T03:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:08,690 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:08,690 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 796efe4cc831df7448a6f85d3ad0d44b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:08,691 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:08,691 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:08,691 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/B is initiating minor compaction (all files) 2024-11-18T03:16:08,691 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/A is initiating minor compaction (all files) 2024-11-18T03:16:08,691 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/B in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,691 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/A in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,691 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/2ca6d7ad5e4544d0b0ef0e2d606f5fee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/17ab3e3d1a5048e6bbaec1d7188a1b73, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/b8770333320045ae8d82df6e46e4cc2c] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.9 K 2024-11-18T03:16:08,691 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/47d6ee900f8c46ceafb74d186b00daaf, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6f5cddf4cd07483c850e544e53a6581f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/032cd025481c464d9bb3ca619857846d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.9 K 2024-11-18T03:16:08,691 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca6d7ad5e4544d0b0ef0e2d606f5fee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1731899766159 2024-11-18T03:16:08,692 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d6ee900f8c46ceafb74d186b00daaf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1731899766159 2024-11-18T03:16:08,693 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ab3e3d1a5048e6bbaec1d7188a1b73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731899766201 2024-11-18T03:16:08,693 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f5cddf4cd07483c850e544e53a6581f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731899766201 2024-11-18T03:16:08,693 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b8770333320045ae8d82df6e46e4cc2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1731899766663 2024-11-18T03:16:08,694 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032cd025481c464d9bb3ca619857846d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1731899766663 2024-11-18T03:16:08,707 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#A#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:08,707 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/f7dd261b419d496f8a08d8422e355ae9 is 50, key is test_row_0/A:col10/1731899766678/Put/seqid=0 2024-11-18T03:16:08,723 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:08,724 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/39955cae09f44ecea4cc46b21b4284aa is 50, key is test_row_0/B:col10/1731899766678/Put/seqid=0 2024-11-18T03:16:08,729 INFO [Thread-1260 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1260. 2024-11-18T03:16:08,729 DEBUG [Thread-1260 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:08,729 DEBUG [Thread-1260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,729 DEBUG [Thread-1260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:08,730 INFO [Thread-1266 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1266. 2024-11-18T03:16:08,730 DEBUG [Thread-1266 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:08,730 DEBUG [Thread-1266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,731 DEBUG [Thread-1266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:08,731 INFO [Thread-1264 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1264. 2024-11-18T03:16:08,731 DEBUG [Thread-1264 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:08,731 DEBUG [Thread-1264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,731 DEBUG [Thread-1264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:08,733 INFO [Thread-1258 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1258. 2024-11-18T03:16:08,733 DEBUG [Thread-1258 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:08,733 DEBUG [Thread-1258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,733 DEBUG [Thread-1258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,733 INFO [Thread-1262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1262. 2024-11-18T03:16:08,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:08,733 DEBUG [Thread-1262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:08,733 DEBUG [Thread-1262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:08,733 DEBUG [Thread-1262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742181_1357 (size=13255) 2024-11-18T03:16:08,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742182_1358 (size=13255) 2024-11-18T03:16:08,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T03:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:08,790 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T03:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/7cf90374ae8d494091dd9e826ba3a45b is 50, key is test_row_0/A:col10/1731899767339/Put/seqid=0 2024-11-18T03:16:08,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742183_1359 (size=12301) 2024-11-18T03:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T03:16:09,138 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/f7dd261b419d496f8a08d8422e355ae9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/f7dd261b419d496f8a08d8422e355ae9 2024-11-18T03:16:09,143 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/A of 796efe4cc831df7448a6f85d3ad0d44b into f7dd261b419d496f8a08d8422e355ae9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:09,143 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:09,143 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/A, priority=13, startTime=1731899768690; duration=0sec 2024-11-18T03:16:09,143 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:09,144 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:A 2024-11-18T03:16:09,144 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:09,144 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/39955cae09f44ecea4cc46b21b4284aa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/39955cae09f44ecea4cc46b21b4284aa 2024-11-18T03:16:09,144 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:09,144 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 796efe4cc831df7448a6f85d3ad0d44b/C is initiating minor compaction (all files) 2024-11-18T03:16:09,144 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 796efe4cc831df7448a6f85d3ad0d44b/C in TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:09,145 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d96dbb9b12984080b969f9cdd7f33a33, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a7cfda5c60384956942df4119787a41d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/bdcda01b57374ef68840338347f8d98e] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp, totalSize=36.9 K 2024-11-18T03:16:09,145 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d96dbb9b12984080b969f9cdd7f33a33, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1731899766159 2024-11-18T03:16:09,145 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7cfda5c60384956942df4119787a41d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731899766201 2024-11-18T03:16:09,145 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdcda01b57374ef68840338347f8d98e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1731899766663 2024-11-18T03:16:09,149 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/B of 796efe4cc831df7448a6f85d3ad0d44b into 39955cae09f44ecea4cc46b21b4284aa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:09,149 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:09,149 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/B, priority=13, startTime=1731899768690; duration=0sec 2024-11-18T03:16:09,149 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:09,149 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:B 2024-11-18T03:16:09,151 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 796efe4cc831df7448a6f85d3ad0d44b#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:09,152 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/78246ff8fc6b4e4699074725cf8e9c13 is 50, key is test_row_0/C:col10/1731899766678/Put/seqid=0 2024-11-18T03:16:09,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742184_1360 (size=13255) 2024-11-18T03:16:09,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/7cf90374ae8d494091dd9e826ba3a45b 2024-11-18T03:16:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/c4ce1f842fd14f6ca5bdedf6b0b72d84 is 50, key is test_row_0/B:col10/1731899767339/Put/seqid=0 2024-11-18T03:16:09,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742185_1361 (size=12301) 2024-11-18T03:16:09,493 INFO [Thread-1249 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1249. 2024-11-18T03:16:09,493 DEBUG [Thread-1249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:09,493 DEBUG [Thread-1249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:09,493 DEBUG [Thread-1249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:09,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:09,495 INFO [Thread-1255 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1255. 2024-11-18T03:16:09,496 DEBUG [Thread-1255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:09,496 DEBUG [Thread-1255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:09,496 DEBUG [Thread-1255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:09,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:09,503 INFO [Thread-1251 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1251. 2024-11-18T03:16:09,504 DEBUG [Thread-1251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:09,504 DEBUG [Thread-1251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:09,504 DEBUG [Thread-1251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:09,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:09,567 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/78246ff8fc6b4e4699074725cf8e9c13 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/78246ff8fc6b4e4699074725cf8e9c13 2024-11-18T03:16:09,583 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 796efe4cc831df7448a6f85d3ad0d44b/C of 796efe4cc831df7448a6f85d3ad0d44b into 78246ff8fc6b4e4699074725cf8e9c13(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:09,583 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:09,583 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., storeName=796efe4cc831df7448a6f85d3ad0d44b/C, priority=13, startTime=1731899768690; duration=0sec 2024-11-18T03:16:09,583 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:09,583 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 796efe4cc831df7448a6f85d3ad0d44b:C 2024-11-18T03:16:09,615 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/c4ce1f842fd14f6ca5bdedf6b0b72d84 2024-11-18T03:16:09,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/ea0b9cfa2476466088b3effeb8a2d437 is 50, key is test_row_0/C:col10/1731899767339/Put/seqid=0 2024-11-18T03:16:09,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742186_1362 (size=12301) 2024-11-18T03:16:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T03:16:10,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/ea0b9cfa2476466088b3effeb8a2d437 2024-11-18T03:16:10,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/7cf90374ae8d494091dd9e826ba3a45b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7cf90374ae8d494091dd9e826ba3a45b 2024-11-18T03:16:10,046 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7cf90374ae8d494091dd9e826ba3a45b, entries=150, sequenceid=677, filesize=12.0 K 2024-11-18T03:16:10,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/c4ce1f842fd14f6ca5bdedf6b0b72d84 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c4ce1f842fd14f6ca5bdedf6b0b72d84 2024-11-18T03:16:10,050 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c4ce1f842fd14f6ca5bdedf6b0b72d84, entries=150, sequenceid=677, filesize=12.0 K 2024-11-18T03:16:10,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/ea0b9cfa2476466088b3effeb8a2d437 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/ea0b9cfa2476466088b3effeb8a2d437 2024-11-18T03:16:10,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/ea0b9cfa2476466088b3effeb8a2d437, entries=150, sequenceid=677, filesize=12.0 K 2024-11-18T03:16:10,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=20.13 KB/20610 for 796efe4cc831df7448a6f85d3ad0d44b in 1264ms, sequenceid=677, compaction requested=false 2024-11-18T03:16:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 796efe4cc831df7448a6f85d3ad0d44b: 2024-11-18T03:16:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-18T03:16:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-18T03:16:10,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-18T03:16:10,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2090 sec 2024-11-18T03:16:10,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.2150 sec 2024-11-18T03:16:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T03:16:12,001 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:12,162 INFO [Thread-1253 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1253. 2024-11-18T03:16:12,162 DEBUG [Thread-1253 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:12,162 DEBUG [Thread-1253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:12,162 DEBUG [Thread-1253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:12,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:12,216 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T03:16:16,353 INFO [Thread-1247 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1247. 2024-11-18T03:16:16,353 DEBUG [Thread-1247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:16,353 DEBUG [Thread-1247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:16,354 DEBUG [Thread-1247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:16,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 161 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2440 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2467 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2525 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2423 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2469 2024-11-18T03:16:16,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T03:16:16,354 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:16:16,354 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:16:16,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:16,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:16,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:16,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-18T03:16:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T03:16:16,358 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899776358"}]},"ts":"1731899776358"} 2024-11-18T03:16:16,360 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T03:16:16,360 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T03:16:16,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:16:16,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, UNASSIGN}] 2024-11-18T03:16:16,362 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, UNASSIGN 2024-11-18T03:16:16,363 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=796efe4cc831df7448a6f85d3ad0d44b, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:16,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, UNASSIGN because future has completed 2024-11-18T03:16:16,364 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:16:16,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:16:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T03:16:16,516 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:16,516 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:16:16,516 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 796efe4cc831df7448a6f85d3ad0d44b, disabling compactions & flushes 2024-11-18T03:16:16,516 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:16,516 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:16,516 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. after waiting 0 ms 2024-11-18T03:16:16,516 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:16,517 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 796efe4cc831df7448a6f85d3ad0d44b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-18T03:16:16,517 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=A 2024-11-18T03:16:16,517 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:16,517 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=B 2024-11-18T03:16:16,517 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:16,517 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 796efe4cc831df7448a6f85d3ad0d44b, store=C 2024-11-18T03:16:16,517 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:16,521 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/cf04b7b704a64344be76869174b3e0a7 is 50, key is test_row_1/A:col10/1731899772161/Put/seqid=0 2024-11-18T03:16:16,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742187_1363 (size=9857) 2024-11-18T03:16:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T03:16:16,925 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/cf04b7b704a64344be76869174b3e0a7 2024-11-18T03:16:16,933 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/975f913889444cd6bc485b98461009f5 is 50, key is test_row_1/B:col10/1731899772161/Put/seqid=0 2024-11-18T03:16:16,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742188_1364 (size=9857) 2024-11-18T03:16:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T03:16:17,344 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/975f913889444cd6bc485b98461009f5 2024-11-18T03:16:17,351 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d6b0cc35c3eb4407bdf915cc060fbdaf is 50, key is test_row_1/C:col10/1731899772161/Put/seqid=0 2024-11-18T03:16:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742189_1365 (size=9857) 2024-11-18T03:16:17,358 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d6b0cc35c3eb4407bdf915cc060fbdaf 2024-11-18T03:16:17,363 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/A/cf04b7b704a64344be76869174b3e0a7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/cf04b7b704a64344be76869174b3e0a7 2024-11-18T03:16:17,368 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/cf04b7b704a64344be76869174b3e0a7, entries=100, sequenceid=688, filesize=9.6 K 2024-11-18T03:16:17,369 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/B/975f913889444cd6bc485b98461009f5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/975f913889444cd6bc485b98461009f5 2024-11-18T03:16:17,378 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/975f913889444cd6bc485b98461009f5, entries=100, sequenceid=688, filesize=9.6 K 2024-11-18T03:16:17,379 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/.tmp/C/d6b0cc35c3eb4407bdf915cc060fbdaf as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d6b0cc35c3eb4407bdf915cc060fbdaf 2024-11-18T03:16:17,385 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d6b0cc35c3eb4407bdf915cc060fbdaf, entries=100, sequenceid=688, filesize=9.6 K 2024-11-18T03:16:17,386 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 796efe4cc831df7448a6f85d3ad0d44b in 869ms, sequenceid=688, compaction requested=true 2024-11-18T03:16:17,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/79c76560dfe94db1b7b0170143313cf4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/e576b933e8b54366ac5244e586c4a021, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d81e4c4be5894267a16745656069bae8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/3ea5801bca4b4b33aa9b81503022d303, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/962ab15d5cab4391ba172ab60f2af322, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/84e89e52dccd4323b04335cc422bcbfa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6becb7156309408fb3891b3bdd3dfe00, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1f4050faec694d1baa97364e8a975761, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/4e30ad8e8e0643a08876cb43c6f5a77a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/fd54e11882fd4d94afdca276aae8ee9a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/0ee9d06f60e541adacfc882708400a6c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/93dc549e3cf3449498cc80abae867606, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/955daa0909f742ec9a82bbd1a7ff6ad6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/8a843ae111444de0b57086769b26f9ba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d252662f12b44717884d658a0eff07fb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1d09ea84e5e4449896a305ecb25a8f69, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/455843319b7c4a1eaff91b52fecc2a30, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/53ce575aec7d47b9832c1913f958759c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/19471183363e42de991a180e57aefe1c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/754c43962677418db9457ce751b37282, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/a29f8bccbbfe468d872595ae8de9783d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6ba4beece2dd400395e83ef81b2e2361, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/73c22020ddcf46df80701886a27981d2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7046cb9903854cdbb9c4d51b121576cb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/5ee2492fa5c142d7a124b98ec5b5f51d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/47d6ee900f8c46ceafb74d186b00daaf, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6f5cddf4cd07483c850e544e53a6581f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/032cd025481c464d9bb3ca619857846d] to archive 2024-11-18T03:16:17,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:16:17,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/79c76560dfe94db1b7b0170143313cf4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/79c76560dfe94db1b7b0170143313cf4 2024-11-18T03:16:17,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/e576b933e8b54366ac5244e586c4a021 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/e576b933e8b54366ac5244e586c4a021 2024-11-18T03:16:17,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d81e4c4be5894267a16745656069bae8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d81e4c4be5894267a16745656069bae8 2024-11-18T03:16:17,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/3ea5801bca4b4b33aa9b81503022d303 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/3ea5801bca4b4b33aa9b81503022d303 2024-11-18T03:16:17,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/962ab15d5cab4391ba172ab60f2af322 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/962ab15d5cab4391ba172ab60f2af322 2024-11-18T03:16:17,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/84e89e52dccd4323b04335cc422bcbfa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/84e89e52dccd4323b04335cc422bcbfa 2024-11-18T03:16:17,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6becb7156309408fb3891b3bdd3dfe00 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6becb7156309408fb3891b3bdd3dfe00 2024-11-18T03:16:17,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1f4050faec694d1baa97364e8a975761 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1f4050faec694d1baa97364e8a975761 2024-11-18T03:16:17,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/4e30ad8e8e0643a08876cb43c6f5a77a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/4e30ad8e8e0643a08876cb43c6f5a77a 2024-11-18T03:16:17,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/fd54e11882fd4d94afdca276aae8ee9a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/fd54e11882fd4d94afdca276aae8ee9a 2024-11-18T03:16:17,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/0ee9d06f60e541adacfc882708400a6c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/0ee9d06f60e541adacfc882708400a6c 2024-11-18T03:16:17,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/93dc549e3cf3449498cc80abae867606 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/93dc549e3cf3449498cc80abae867606 2024-11-18T03:16:17,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/955daa0909f742ec9a82bbd1a7ff6ad6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/955daa0909f742ec9a82bbd1a7ff6ad6 2024-11-18T03:16:17,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/8a843ae111444de0b57086769b26f9ba to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/8a843ae111444de0b57086769b26f9ba 2024-11-18T03:16:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T03:16:17,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d252662f12b44717884d658a0eff07fb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/d252662f12b44717884d658a0eff07fb 2024-11-18T03:16:17,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1d09ea84e5e4449896a305ecb25a8f69 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/1d09ea84e5e4449896a305ecb25a8f69 2024-11-18T03:16:17,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/455843319b7c4a1eaff91b52fecc2a30 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/455843319b7c4a1eaff91b52fecc2a30 2024-11-18T03:16:17,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/53ce575aec7d47b9832c1913f958759c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/53ce575aec7d47b9832c1913f958759c 2024-11-18T03:16:17,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/19471183363e42de991a180e57aefe1c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/19471183363e42de991a180e57aefe1c 2024-11-18T03:16:17,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/754c43962677418db9457ce751b37282 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/754c43962677418db9457ce751b37282 2024-11-18T03:16:17,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/a29f8bccbbfe468d872595ae8de9783d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/a29f8bccbbfe468d872595ae8de9783d 2024-11-18T03:16:17,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6ba4beece2dd400395e83ef81b2e2361 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6ba4beece2dd400395e83ef81b2e2361 2024-11-18T03:16:17,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/73c22020ddcf46df80701886a27981d2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/73c22020ddcf46df80701886a27981d2 2024-11-18T03:16:17,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7046cb9903854cdbb9c4d51b121576cb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7046cb9903854cdbb9c4d51b121576cb 2024-11-18T03:16:17,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/5ee2492fa5c142d7a124b98ec5b5f51d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/5ee2492fa5c142d7a124b98ec5b5f51d 2024-11-18T03:16:17,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/47d6ee900f8c46ceafb74d186b00daaf to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/47d6ee900f8c46ceafb74d186b00daaf 2024-11-18T03:16:17,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6f5cddf4cd07483c850e544e53a6581f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/6f5cddf4cd07483c850e544e53a6581f 2024-11-18T03:16:17,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/032cd025481c464d9bb3ca619857846d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/032cd025481c464d9bb3ca619857846d 2024-11-18T03:16:17,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/43ce3261582b48268be2305579f3ce76, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faf5fae0927446ef93f1563f3e16e2bb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/26e7e16602d84a1287ec1fe1647f1409, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/7d7c3a54c5734718bbbf2d356b6f2ec6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/291bff8b2966410381adae89b4851ec9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c719f3e31a354b72b1511e0c0ec897ac, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/d6a69aef3ec5447e988bf473c7c230dc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/616042c73a474714814e86408805a9e5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3e50bd9712744c6f870a91e9df680221, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/89a4c37b97914da0b85f4c14936d5383, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3dfdfa331f51434582db02a329d817fa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/093cbfe4c26043f0a1a159a4299c7104, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/1332582544394d80a25924553fcaba21, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/e8406fef5b064ccd82256adf513d9d45, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/94ce2476c14446de9625c15b83c3cedd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/394012a787a24f4fb7962767f58ffe4b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/40a5c4685c484ca0ad0fe2d66e76f888, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/8275a1e3c230444aab57bdf86f71be14, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/34a62f98f72a469fa163307e95746b75, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faa6664351314042b2290832c8d6bde0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/742514c894aa47939843ec32e8c70d74, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/cf8369524bdc48868a56f937cead5056, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/66a15c03d6274a0c9baade7ac2e59e32, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/2ca6d7ad5e4544d0b0ef0e2d606f5fee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/0723e06bb5f341b4a2323c5d66f156f8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/17ab3e3d1a5048e6bbaec1d7188a1b73, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/b8770333320045ae8d82df6e46e4cc2c] to archive 2024-11-18T03:16:17,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:16:17,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/43ce3261582b48268be2305579f3ce76 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/43ce3261582b48268be2305579f3ce76 2024-11-18T03:16:17,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faf5fae0927446ef93f1563f3e16e2bb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faf5fae0927446ef93f1563f3e16e2bb 2024-11-18T03:16:17,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/26e7e16602d84a1287ec1fe1647f1409 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/26e7e16602d84a1287ec1fe1647f1409 2024-11-18T03:16:17,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/7d7c3a54c5734718bbbf2d356b6f2ec6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/7d7c3a54c5734718bbbf2d356b6f2ec6 2024-11-18T03:16:17,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/ec6649f5d35b4b1b9ca5a2f4c8e377e2 2024-11-18T03:16:17,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/291bff8b2966410381adae89b4851ec9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/291bff8b2966410381adae89b4851ec9 2024-11-18T03:16:17,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c719f3e31a354b72b1511e0c0ec897ac to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c719f3e31a354b72b1511e0c0ec897ac 2024-11-18T03:16:17,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/d6a69aef3ec5447e988bf473c7c230dc to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/d6a69aef3ec5447e988bf473c7c230dc 2024-11-18T03:16:17,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/616042c73a474714814e86408805a9e5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/616042c73a474714814e86408805a9e5 2024-11-18T03:16:17,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3e50bd9712744c6f870a91e9df680221 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3e50bd9712744c6f870a91e9df680221 2024-11-18T03:16:17,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/89a4c37b97914da0b85f4c14936d5383 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/89a4c37b97914da0b85f4c14936d5383 2024-11-18T03:16:17,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3dfdfa331f51434582db02a329d817fa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/3dfdfa331f51434582db02a329d817fa 2024-11-18T03:16:17,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/093cbfe4c26043f0a1a159a4299c7104 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/093cbfe4c26043f0a1a159a4299c7104 2024-11-18T03:16:17,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/1332582544394d80a25924553fcaba21 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/1332582544394d80a25924553fcaba21 2024-11-18T03:16:17,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/e8406fef5b064ccd82256adf513d9d45 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/e8406fef5b064ccd82256adf513d9d45 2024-11-18T03:16:17,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/94ce2476c14446de9625c15b83c3cedd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/94ce2476c14446de9625c15b83c3cedd 2024-11-18T03:16:17,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/394012a787a24f4fb7962767f58ffe4b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/394012a787a24f4fb7962767f58ffe4b 2024-11-18T03:16:17,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/40a5c4685c484ca0ad0fe2d66e76f888 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/40a5c4685c484ca0ad0fe2d66e76f888 2024-11-18T03:16:17,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/8275a1e3c230444aab57bdf86f71be14 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/8275a1e3c230444aab57bdf86f71be14 2024-11-18T03:16:17,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/34a62f98f72a469fa163307e95746b75 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/34a62f98f72a469fa163307e95746b75 2024-11-18T03:16:17,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faa6664351314042b2290832c8d6bde0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/faa6664351314042b2290832c8d6bde0 2024-11-18T03:16:17,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/742514c894aa47939843ec32e8c70d74 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/742514c894aa47939843ec32e8c70d74 2024-11-18T03:16:17,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/cf8369524bdc48868a56f937cead5056 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/cf8369524bdc48868a56f937cead5056 2024-11-18T03:16:17,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/66a15c03d6274a0c9baade7ac2e59e32 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/66a15c03d6274a0c9baade7ac2e59e32 2024-11-18T03:16:17,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/2ca6d7ad5e4544d0b0ef0e2d606f5fee to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/2ca6d7ad5e4544d0b0ef0e2d606f5fee 2024-11-18T03:16:17,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/0723e06bb5f341b4a2323c5d66f156f8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/0723e06bb5f341b4a2323c5d66f156f8 2024-11-18T03:16:17,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/17ab3e3d1a5048e6bbaec1d7188a1b73 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/17ab3e3d1a5048e6bbaec1d7188a1b73 2024-11-18T03:16:17,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/b8770333320045ae8d82df6e46e4cc2c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/b8770333320045ae8d82df6e46e4cc2c 2024-11-18T03:16:17,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f3ef7a79d990401bac4d7386f94cb288, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/39edec4daca342ea8791bd9952bb85c8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7d53b46c95f44d7496d090cd0d612de2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/fe076de632e84f82a8ebde64da3f2d82, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/aeda347f7fb64522b39b618bc1b39062, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7c3d8762ece14f5e8ffad3108382bf58, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/af30865c3b0646edbc4c39ef10ab469d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/5b412efa4c914d5d9ae6b0684c9405e8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/24fa5c8fca4245f6a2d0811d1d3654ce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a53ec05d331844598f96ae46d161208f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/708984eaa5304507a15f491c6ae04850, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8d933121e1ef4396a37377e786794c7b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7dd5705e79a94425b45c47699e6d3ded, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8e8939b016844f35902422c25825ba0c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f0f2498a91b54d7593582bde77100aee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7888dd7b412048b79b2f9eabd3c0a47d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4d1280ebc61e4b83b15aa697a8a83d3f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/22b0081927c34acb811940e287649691, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/e21204c13fa74c1b9cf6ab4614cbaa15, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/49828163576846219100081bb2a37792, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/b560a62f7c524562b4f6bf96c0f52afa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4b5365db823c4a9f8dbef3992223bdbf, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/3876ce2d7ff0437e881f6886e8115fdb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/05d370253b3842d9940df2d1a929651a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d96dbb9b12984080b969f9cdd7f33a33, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d9f95ee2d40b41e29172859d18680a7b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a7cfda5c60384956942df4119787a41d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/bdcda01b57374ef68840338347f8d98e] to archive 2024-11-18T03:16:17,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:16:17,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f3ef7a79d990401bac4d7386f94cb288 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f3ef7a79d990401bac4d7386f94cb288 2024-11-18T03:16:17,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/39edec4daca342ea8791bd9952bb85c8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/39edec4daca342ea8791bd9952bb85c8 2024-11-18T03:16:17,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7d53b46c95f44d7496d090cd0d612de2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7d53b46c95f44d7496d090cd0d612de2 2024-11-18T03:16:17,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/fe076de632e84f82a8ebde64da3f2d82 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/fe076de632e84f82a8ebde64da3f2d82 2024-11-18T03:16:17,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/aeda347f7fb64522b39b618bc1b39062 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/aeda347f7fb64522b39b618bc1b39062 2024-11-18T03:16:17,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7c3d8762ece14f5e8ffad3108382bf58 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7c3d8762ece14f5e8ffad3108382bf58 2024-11-18T03:16:17,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/af30865c3b0646edbc4c39ef10ab469d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/af30865c3b0646edbc4c39ef10ab469d 2024-11-18T03:16:17,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/5b412efa4c914d5d9ae6b0684c9405e8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/5b412efa4c914d5d9ae6b0684c9405e8 2024-11-18T03:16:17,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/24fa5c8fca4245f6a2d0811d1d3654ce to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/24fa5c8fca4245f6a2d0811d1d3654ce 2024-11-18T03:16:17,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a53ec05d331844598f96ae46d161208f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a53ec05d331844598f96ae46d161208f 2024-11-18T03:16:17,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/708984eaa5304507a15f491c6ae04850 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/708984eaa5304507a15f491c6ae04850 2024-11-18T03:16:17,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8d933121e1ef4396a37377e786794c7b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8d933121e1ef4396a37377e786794c7b 2024-11-18T03:16:17,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7dd5705e79a94425b45c47699e6d3ded to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7dd5705e79a94425b45c47699e6d3ded 2024-11-18T03:16:17,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8e8939b016844f35902422c25825ba0c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/8e8939b016844f35902422c25825ba0c 2024-11-18T03:16:17,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f0f2498a91b54d7593582bde77100aee to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/f0f2498a91b54d7593582bde77100aee 2024-11-18T03:16:17,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7888dd7b412048b79b2f9eabd3c0a47d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/7888dd7b412048b79b2f9eabd3c0a47d 2024-11-18T03:16:17,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4d1280ebc61e4b83b15aa697a8a83d3f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4d1280ebc61e4b83b15aa697a8a83d3f 2024-11-18T03:16:17,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/22b0081927c34acb811940e287649691 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/22b0081927c34acb811940e287649691 2024-11-18T03:16:17,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/e21204c13fa74c1b9cf6ab4614cbaa15 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/e21204c13fa74c1b9cf6ab4614cbaa15 2024-11-18T03:16:17,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/49828163576846219100081bb2a37792 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/49828163576846219100081bb2a37792 2024-11-18T03:16:17,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/b560a62f7c524562b4f6bf96c0f52afa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/b560a62f7c524562b4f6bf96c0f52afa 2024-11-18T03:16:17,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4b5365db823c4a9f8dbef3992223bdbf to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/4b5365db823c4a9f8dbef3992223bdbf 2024-11-18T03:16:17,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/3876ce2d7ff0437e881f6886e8115fdb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/3876ce2d7ff0437e881f6886e8115fdb 2024-11-18T03:16:17,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/05d370253b3842d9940df2d1a929651a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/05d370253b3842d9940df2d1a929651a 2024-11-18T03:16:17,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d96dbb9b12984080b969f9cdd7f33a33 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d96dbb9b12984080b969f9cdd7f33a33 2024-11-18T03:16:17,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d9f95ee2d40b41e29172859d18680a7b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d9f95ee2d40b41e29172859d18680a7b 2024-11-18T03:16:18,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a7cfda5c60384956942df4119787a41d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/a7cfda5c60384956942df4119787a41d 2024-11-18T03:16:18,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/bdcda01b57374ef68840338347f8d98e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/bdcda01b57374ef68840338347f8d98e 2024-11-18T03:16:18,060 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/recovered.edits/691.seqid, newMaxSeqId=691, maxSeqId=1 2024-11-18T03:16:18,066 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. 2024-11-18T03:16:18,066 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 796efe4cc831df7448a6f85d3ad0d44b: Waiting for close lock at 1731899776516Running coprocessor pre-close hooks at 1731899776516Disabling compacts and flushes for region at 1731899776516Disabling writes for close at 1731899776516Obtaining lock to block concurrent updates at 1731899776517 (+1 ms)Preparing flush snapshotting stores in 796efe4cc831df7448a6f85d3ad0d44b at 1731899776517Finished memstore snapshotting TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731899776517Flushing stores of TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b. at 1731899776518 (+1 ms)Flushing 796efe4cc831df7448a6f85d3ad0d44b/A: creating writer at 1731899776518Flushing 796efe4cc831df7448a6f85d3ad0d44b/A: appending metadata at 1731899776520 (+2 ms)Flushing 796efe4cc831df7448a6f85d3ad0d44b/A: closing flushed file at 1731899776520Flushing 796efe4cc831df7448a6f85d3ad0d44b/B: creating writer at 1731899776930 (+410 ms)Flushing 796efe4cc831df7448a6f85d3ad0d44b/B: appending metadata at 1731899776932 (+2 ms)Flushing 796efe4cc831df7448a6f85d3ad0d44b/B: closing flushed file at 1731899776933 (+1 ms)Flushing 796efe4cc831df7448a6f85d3ad0d44b/C: creating writer at 1731899777348 (+415 ms)Flushing 796efe4cc831df7448a6f85d3ad0d44b/C: appending metadata at 1731899777351 (+3 ms)Flushing 796efe4cc831df7448a6f85d3ad0d44b/C: closing flushed file at 1731899777351Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69ac6e45: reopening flushed file at 1731899777362 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47478b74: reopening flushed file at 1731899777368 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c24c8c9: reopening flushed file at 1731899777378 (+10 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 796efe4cc831df7448a6f85d3ad0d44b in 869ms, sequenceid=688, compaction requested=true at 1731899777386 (+8 ms)Writing region close event to WAL at 1731899778036 (+650 ms)Running coprocessor post-close hooks at 1731899778066 (+30 ms)Closed at 1731899778066 2024-11-18T03:16:18,088 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:18,096 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=796efe4cc831df7448a6f85d3ad0d44b, regionState=CLOSED 2024-11-18T03:16:18,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:16:18,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-18T03:16:18,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 796efe4cc831df7448a6f85d3ad0d44b, server=fd862c3a04f9,38549,1731899684479 in 1.7430 sec 2024-11-18T03:16:18,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-18T03:16:18,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=796efe4cc831df7448a6f85d3ad0d44b, UNASSIGN in 1.7490 sec 2024-11-18T03:16:18,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-18T03:16:18,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7560 sec 2024-11-18T03:16:18,131 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899778131"}]},"ts":"1731899778131"} 2024-11-18T03:16:18,137 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T03:16:18,137 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T03:16:18,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7830 sec 2024-11-18T03:16:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T03:16:18,505 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:18,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-18T03:16:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:18,528 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T03:16:18,533 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:18,541 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:18,596 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/recovered.edits] 2024-11-18T03:16:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T03:16:18,642 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7cf90374ae8d494091dd9e826ba3a45b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/7cf90374ae8d494091dd9e826ba3a45b 2024-11-18T03:16:18,660 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/cf04b7b704a64344be76869174b3e0a7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/cf04b7b704a64344be76869174b3e0a7 2024-11-18T03:16:18,687 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/f7dd261b419d496f8a08d8422e355ae9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/A/f7dd261b419d496f8a08d8422e355ae9 2024-11-18T03:16:18,714 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/39955cae09f44ecea4cc46b21b4284aa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/39955cae09f44ecea4cc46b21b4284aa 2024-11-18T03:16:18,735 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/975f913889444cd6bc485b98461009f5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/975f913889444cd6bc485b98461009f5 2024-11-18T03:16:18,760 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c4ce1f842fd14f6ca5bdedf6b0b72d84 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/B/c4ce1f842fd14f6ca5bdedf6b0b72d84 2024-11-18T03:16:18,777 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/78246ff8fc6b4e4699074725cf8e9c13 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/78246ff8fc6b4e4699074725cf8e9c13 2024-11-18T03:16:18,779 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d6b0cc35c3eb4407bdf915cc060fbdaf to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/d6b0cc35c3eb4407bdf915cc060fbdaf 2024-11-18T03:16:18,790 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/ea0b9cfa2476466088b3effeb8a2d437 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/C/ea0b9cfa2476466088b3effeb8a2d437 2024-11-18T03:16:18,800 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/recovered.edits/691.seqid to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b/recovered.edits/691.seqid 2024-11-18T03:16:18,804 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/796efe4cc831df7448a6f85d3ad0d44b 2024-11-18T03:16:18,805 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T03:16:18,813 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:18,824 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T03:16:18,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T03:16:18,836 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:18,837 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T03:16:18,837 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731899778837"}]},"ts":"9223372036854775807"} 2024-11-18T03:16:18,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T03:16:18,845 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 796efe4cc831df7448a6f85d3ad0d44b, NAME => 'TestAcidGuarantees,,1731899746524.796efe4cc831df7448a6f85d3ad0d44b.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T03:16:18,845 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T03:16:18,846 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731899778846"}]},"ts":"9223372036854775807"} 2024-11-18T03:16:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T03:16:18,857 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T03:16:18,868 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:18,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 363 msec 2024-11-18T03:16:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T03:16:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T03:16:19,168 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:19,192 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=447 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=816 (was 789) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3434 (was 3263) - AvailableMemoryMB LEAK? - 2024-11-18T03:16:19,207 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=816, ProcessCount=11, AvailableMemoryMB=3433 2024-11-18T03:16:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:16:19,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:16:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:19,227 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:16:19,227 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:19,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-18T03:16:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T03:16:19,230 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:16:19,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742190_1366 (size=1061) 2024-11-18T03:16:19,317 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:16:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T03:16:19,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742191_1367 (size=53) 2024-11-18T03:16:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T03:16:19,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:16:19,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 70fa1092abe5ed71c7b0809f585c5469, disabling compactions & flushes 2024-11-18T03:16:19,769 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:19,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:19,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. after waiting 0 ms 2024-11-18T03:16:19,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:19,769 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:19,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 70fa1092abe5ed71c7b0809f585c5469: Waiting for close lock at 1731899779769Disabling compacts and flushes for region at 1731899779769Disabling writes for close at 1731899779769Writing region close event to WAL at 1731899779769Closed at 1731899779769 2024-11-18T03:16:19,775 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:16:19,776 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731899779775"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731899779775"}]},"ts":"1731899779775"} 2024-11-18T03:16:19,792 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:16:19,797 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:16:19,797 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899779797"}]},"ts":"1731899779797"} 2024-11-18T03:16:19,812 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T03:16:19,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, ASSIGN}] 2024-11-18T03:16:19,826 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, ASSIGN 2024-11-18T03:16:19,832 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, ASSIGN; state=OFFLINE, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=false 2024-11-18T03:16:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T03:16:19,992 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:20,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, ASSIGN because future has completed 2024-11-18T03:16:20,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:16:20,161 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,162 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:16:20,162 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,162 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:16:20,162 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,162 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,164 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,172 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:20,172 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70fa1092abe5ed71c7b0809f585c5469 columnFamilyName A 2024-11-18T03:16:20,172 DEBUG [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:20,174 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(327): Store=70fa1092abe5ed71c7b0809f585c5469/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:20,174 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,181 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:20,181 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70fa1092abe5ed71c7b0809f585c5469 columnFamilyName B 2024-11-18T03:16:20,181 DEBUG [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:20,186 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(327): Store=70fa1092abe5ed71c7b0809f585c5469/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:20,186 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,191 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:20,192 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70fa1092abe5ed71c7b0809f585c5469 columnFamilyName C 2024-11-18T03:16:20,192 DEBUG [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:20,194 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(327): Store=70fa1092abe5ed71c7b0809f585c5469/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:20,195 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,195 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,197 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,198 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,199 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,199 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,201 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:16:20,203 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,224 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:16:20,225 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 70fa1092abe5ed71c7b0809f585c5469; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65281158, jitterRate=-0.027234941720962524}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:16:20,225 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,226 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 70fa1092abe5ed71c7b0809f585c5469: Running coprocessor pre-open hook at 1731899780162Writing region info on filesystem at 1731899780162Initializing all the Stores at 1731899780163 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899780163Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899780164 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899780164Cleaning up temporary data from old regions at 1731899780199 (+35 ms)Running coprocessor post-open hooks at 1731899780225 (+26 ms)Region opened successfully at 1731899780226 (+1 ms) 2024-11-18T03:16:20,232 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., pid=89, masterSystemTime=1731899780158 2024-11-18T03:16:20,238 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,238 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,241 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=OPEN, openSeqNum=2, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:20,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:16:20,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-18T03:16:20,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 in 242 msec 2024-11-18T03:16:20,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-18T03:16:20,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, ASSIGN in 438 msec 2024-11-18T03:16:20,258 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:16:20,258 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899780258"}]},"ts":"1731899780258"} 2024-11-18T03:16:20,260 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T03:16:20,261 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:16:20,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.0510 sec 2024-11-18T03:16:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T03:16:20,381 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:20,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e89b04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:20,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:20,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:20,384 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9bb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:20,386 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d577b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:20,387 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:20,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:20,393 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:16:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fd862c3a04f9,45051,1731899683325 2024-11-18T03:16:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1210c5e6 2024-11-18T03:16:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:16:20,395 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:16:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:16:20,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:16:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742192_1368 (size=1097) 2024-11-18T03:16:20,471 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T03:16:20,471 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T03:16:20,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:16:20,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, REOPEN/MOVE}] 2024-11-18T03:16:20,486 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, REOPEN/MOVE 2024-11-18T03:16:20,494 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:20,495 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=fd862c3a04f9,38549,1731899684479, table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T03:16:20,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, REOPEN/MOVE because future has completed 2024-11-18T03:16:20,500 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:16:20,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:16:20,660 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,660 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:16:20,660 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 70fa1092abe5ed71c7b0809f585c5469, disabling compactions & flushes 2024-11-18T03:16:20,660 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,660 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,660 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. after waiting 0 ms 2024-11-18T03:16:20,660 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,664 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T03:16:20,665 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:20,665 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 70fa1092abe5ed71c7b0809f585c5469: Waiting for close lock at 1731899780660Running coprocessor pre-close hooks at 1731899780660Disabling compacts and flushes for region at 1731899780660Disabling writes for close at 1731899780660Writing region close event to WAL at 1731899780660Running coprocessor post-close hooks at 1731899780665 (+5 ms)Closed at 1731899780665 2024-11-18T03:16:20,665 WARN [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 70fa1092abe5ed71c7b0809f585c5469 to self. 2024-11-18T03:16:20,667 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:20,668 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=CLOSED 2024-11-18T03:16:20,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:16:20,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-18T03:16:20,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 in 171 msec 2024-11-18T03:16:20,678 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, REOPEN/MOVE; state=CLOSED, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=true 2024-11-18T03:16:20,829 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:20,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, REOPEN/MOVE because future has completed 2024-11-18T03:16:20,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:16:21,009 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,009 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:16:21,009 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,009 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:16:21,010 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,010 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,011 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,012 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:21,012 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70fa1092abe5ed71c7b0809f585c5469 columnFamilyName A 2024-11-18T03:16:21,016 DEBUG [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:21,016 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(327): Store=70fa1092abe5ed71c7b0809f585c5469/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:21,017 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,018 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:21,018 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70fa1092abe5ed71c7b0809f585c5469 columnFamilyName B 2024-11-18T03:16:21,018 DEBUG [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:21,020 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(327): Store=70fa1092abe5ed71c7b0809f585c5469/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:21,020 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,026 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:21,026 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70fa1092abe5ed71c7b0809f585c5469 columnFamilyName C 2024-11-18T03:16:21,026 DEBUG [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:21,030 INFO [StoreOpener-70fa1092abe5ed71c7b0809f585c5469-1 {}] regionserver.HStore(327): Store=70fa1092abe5ed71c7b0809f585c5469/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:21,032 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,032 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,033 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,039 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,045 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,045 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,048 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:16:21,052 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,057 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 70fa1092abe5ed71c7b0809f585c5469; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61075700, jitterRate=-0.0899011492729187}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:16:21,057 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,058 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 70fa1092abe5ed71c7b0809f585c5469: Running coprocessor pre-open hook at 1731899781010Writing region info on filesystem at 1731899781010Initializing all the Stores at 1731899781011 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899781011Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899781011Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899781011Cleaning up temporary data from old regions at 1731899781045 (+34 ms)Running coprocessor post-open hooks at 1731899781057 (+12 ms)Region opened successfully at 1731899781058 (+1 ms) 2024-11-18T03:16:21,061 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., pid=94, masterSystemTime=1731899780988 2024-11-18T03:16:21,065 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,065 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,066 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=OPEN, openSeqNum=5, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:16:21,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-18T03:16:21,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 in 237 msec 2024-11-18T03:16:21,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-18T03:16:21,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, REOPEN/MOVE in 595 msec 2024-11-18T03:16:21,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-18T03:16:21,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 612 msec 2024-11-18T03:16:21,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 696 msec 2024-11-18T03:16:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-18T03:16:21,102 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:21,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746e823f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,104 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c71b94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,108 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0a56f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acb64fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,124 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f989c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,126 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3009afc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5513552f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,136 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6294c0a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,139 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe66c0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5745c2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,152 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe94ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673d84b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac50c9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,170 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6e1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724b6785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4fb7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,184 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bad0749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,187 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5933c267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1483dc7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dccf358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,223 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5500e2df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78bf9cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,227 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641c0529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,232 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425565aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3074b693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac0fa8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ffa1d67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4767acc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:21,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:21,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f4786f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:21,249 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e82e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:21,258 DEBUG [Thread-1668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,261 DEBUG [Thread-1666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,263 DEBUG [Thread-1675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,264 DEBUG [Thread-1677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:21,269 DEBUG [Thread-1679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-18T03:16:21,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,274 DEBUG [Thread-1662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,274 DEBUG [Thread-1673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,275 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:21,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:21,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:21,277 DEBUG [Thread-1673 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,278 DEBUG [Thread-1673 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T03:16:21,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,291 DEBUG [Thread-1660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,293 DEBUG [Thread-1660 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,322 DEBUG [Thread-1664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:16:21,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:21,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:21,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:21,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:21,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:21,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,375 DEBUG [Thread-1671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:21,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T03:16:21,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b939a9d0e0304c11b03c9d885d150bd9_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899781307/Put/seqid=0 2024-11-18T03:16:21,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742193_1369 (size=12154) 2024-11-18T03:16:21,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T03:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899781497, max timestamp=1731899781623Num uniques 859; 2024-11-18T03:16:21,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:16:21,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899781497, max timestamp=1731899781623Num uniques 859; 2024-11-18T03:16:21,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:16:21,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899781497, max timestamp=1731899781623Num uniques 859; 2024-11-18T03:16:21,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:21,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899841720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899841723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899841724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899841745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899841748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,835 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899841852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899841856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899841863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b939a9d0e0304c11b03c9d885d150bd9_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b939a9d0e0304c11b03c9d885d150bd9_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:21,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/b44ccb4cb3684ed19f2ff39e573b039d, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:21,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/b44ccb4cb3684ed19f2ff39e573b039d is 175, key is test_row_0/A:col10/1731899781307/Put/seqid=0 2024-11-18T03:16:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899841883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899841883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:21,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742194_1370 (size=30955) 2024-11-18T03:16:21,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/b44ccb4cb3684ed19f2ff39e573b039d 2024-11-18T03:16:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T03:16:21,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:21,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:21,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/d2bddd611f30487d8d7ca6bfaac43951 is 50, key is test_row_0/B:col10/1731899781307/Put/seqid=0 2024-11-18T03:16:22,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742195_1371 (size=12001) 2024-11-18T03:16:22,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/d2bddd611f30487d8d7ca6bfaac43951 2024-11-18T03:16:22,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899842072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899842072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:22,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899842081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899842102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899842104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3d86f852969f4fc79eedc756f424b532 is 50, key is test_row_0/C:col10/1731899781307/Put/seqid=0 2024-11-18T03:16:22,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742196_1372 (size=12001) 2024-11-18T03:16:22,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:22,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:22,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899842391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899842392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:22,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899842404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T03:16:22,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899842434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899842435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:22,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:22,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3d86f852969f4fc79eedc756f424b532 2024-11-18T03:16:22,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/b44ccb4cb3684ed19f2ff39e573b039d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/b44ccb4cb3684ed19f2ff39e573b039d 2024-11-18T03:16:22,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/b44ccb4cb3684ed19f2ff39e573b039d, entries=150, sequenceid=14, filesize=30.2 K 2024-11-18T03:16:22,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/d2bddd611f30487d8d7ca6bfaac43951 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d2bddd611f30487d8d7ca6bfaac43951 2024-11-18T03:16:22,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:22,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:22,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:22,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d2bddd611f30487d8d7ca6bfaac43951, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T03:16:22,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3d86f852969f4fc79eedc756f424b532 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d86f852969f4fc79eedc756f424b532 2024-11-18T03:16:22,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d86f852969f4fc79eedc756f424b532, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T03:16:22,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 70fa1092abe5ed71c7b0809f585c5469 in 1481ms, sequenceid=14, compaction requested=false 2024-11-18T03:16:22,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:22,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T03:16:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:22,897 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-18T03:16:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bcc0d9b81a754e7ca65bdd4a49db4fbe_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899781717/Put/seqid=0 2024-11-18T03:16:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:22,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:22,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742197_1373 (size=12154) 2024-11-18T03:16:22,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:22,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899842952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899842965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,983 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bcc0d9b81a754e7ca65bdd4a49db4fbe_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bcc0d9b81a754e7ca65bdd4a49db4fbe_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/19c6ea17b9b943158d6ec48cc4a164a0, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:22,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/19c6ea17b9b943158d6ec48cc4a164a0 is 175, key is test_row_0/A:col10/1731899781717/Put/seqid=0 2024-11-18T03:16:22,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742198_1374 (size=30955) 2024-11-18T03:16:22,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899842979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899842981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:22,996 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/19c6ea17b9b943158d6ec48cc4a164a0 2024-11-18T03:16:23,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899842984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/39d4fbba02ae4621927fefbd1d783f51 is 50, key is test_row_0/B:col10/1731899781717/Put/seqid=0 2024-11-18T03:16:23,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742199_1375 (size=12001) 2024-11-18T03:16:23,077 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/39d4fbba02ae4621927fefbd1d783f51 2024-11-18T03:16:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/8348b4a1c0a1426a9a159913690c9451 is 50, key is test_row_0/C:col10/1731899781717/Put/seqid=0 2024-11-18T03:16:23,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899843091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899843091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899843101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899843101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899843111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742200_1376 (size=12001) 2024-11-18T03:16:23,152 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/8348b4a1c0a1426a9a159913690c9451 2024-11-18T03:16:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/19c6ea17b9b943158d6ec48cc4a164a0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/19c6ea17b9b943158d6ec48cc4a164a0 2024-11-18T03:16:23,190 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/19c6ea17b9b943158d6ec48cc4a164a0, entries=150, sequenceid=50, filesize=30.2 K 2024-11-18T03:16:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/39d4fbba02ae4621927fefbd1d783f51 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/39d4fbba02ae4621927fefbd1d783f51 2024-11-18T03:16:23,200 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/39d4fbba02ae4621927fefbd1d783f51, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T03:16:23,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/8348b4a1c0a1426a9a159913690c9451 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8348b4a1c0a1426a9a159913690c9451 2024-11-18T03:16:23,209 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8348b4a1c0a1426a9a159913690c9451, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T03:16:23,217 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=53.67 KB/54960 for 70fa1092abe5ed71c7b0809f585c5469 in 320ms, sequenceid=50, compaction requested=false 2024-11-18T03:16:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-18T03:16:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-18T03:16:23,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-18T03:16:23,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-18T03:16:23,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9570 sec 2024-11-18T03:16:23,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T03:16:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:23,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e280d63c74a94bdca581691fe56ee712_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899782936/Put/seqid=0 2024-11-18T03:16:23,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742201_1377 (size=14594) 2024-11-18T03:16:23,439 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T03:16:23,457 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:23,468 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-18T03:16:23,477 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T03:16:23,478 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e280d63c74a94bdca581691fe56ee712_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e280d63c74a94bdca581691fe56ee712_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:23,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:23,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:23,489 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/0586d60dc9224a86930d68810eea8bcb, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:23,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/0586d60dc9224a86930d68810eea8bcb is 175, key is test_row_0/A:col10/1731899782936/Put/seqid=0 2024-11-18T03:16:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899843436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742202_1378 (size=39549) 2024-11-18T03:16:23,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/0586d60dc9224a86930d68810eea8bcb 2024-11-18T03:16:23,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899843485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899843486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899843509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899843517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/16b467701c574cb3a01b7977412ef1bd is 50, key is test_row_0/B:col10/1731899782936/Put/seqid=0 2024-11-18T03:16:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T03:16:23,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742203_1379 (size=12001) 2024-11-18T03:16:23,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/16b467701c574cb3a01b7977412ef1bd 2024-11-18T03:16:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899843624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T03:16:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:23,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,649 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:16:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899843653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899843656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/4f3ec6359f9f472a9d331c513ad97b40 is 50, key is test_row_0/C:col10/1731899782936/Put/seqid=0 2024-11-18T03:16:23,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899843682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899843684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742204_1380 (size=12001) 2024-11-18T03:16:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T03:16:23,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T03:16:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:23,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899843844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899843892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899843904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899843931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:23,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899843932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:23,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:23,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T03:16:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:23,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T03:16:24,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T03:16:24,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:24,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:24,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:24,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:24,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:24,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/4f3ec6359f9f472a9d331c513ad97b40 2024-11-18T03:16:24,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899844171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/0586d60dc9224a86930d68810eea8bcb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/0586d60dc9224a86930d68810eea8bcb 2024-11-18T03:16:24,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/0586d60dc9224a86930d68810eea8bcb, entries=200, sequenceid=62, filesize=38.6 K 2024-11-18T03:16:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/16b467701c574cb3a01b7977412ef1bd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/16b467701c574cb3a01b7977412ef1bd 2024-11-18T03:16:24,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899844231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899844232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/16b467701c574cb3a01b7977412ef1bd, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T03:16:24,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/4f3ec6359f9f472a9d331c513ad97b40 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/4f3ec6359f9f472a9d331c513ad97b40 2024-11-18T03:16:24,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899844251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899844251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/4f3ec6359f9f472a9d331c513ad97b40, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T03:16:24,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 70fa1092abe5ed71c7b0809f585c5469 in 933ms, sequenceid=62, compaction requested=true 2024-11-18T03:16:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:24,274 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:24,274 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:24,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:24,281 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:24,281 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:24,281 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:24,281 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/b44ccb4cb3684ed19f2ff39e573b039d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/19c6ea17b9b943158d6ec48cc4a164a0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/0586d60dc9224a86930d68810eea8bcb] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=99.1 K 2024-11-18T03:16:24,281 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:24,281 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/b44ccb4cb3684ed19f2ff39e573b039d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/19c6ea17b9b943158d6ec48cc4a164a0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/0586d60dc9224a86930d68810eea8bcb] 2024-11-18T03:16:24,282 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:24,282 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:24,282 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:24,282 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d2bddd611f30487d8d7ca6bfaac43951, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/39d4fbba02ae4621927fefbd1d783f51, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/16b467701c574cb3a01b7977412ef1bd] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=35.2 K 2024-11-18T03:16:24,282 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting b44ccb4cb3684ed19f2ff39e573b039d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731899781307 2024-11-18T03:16:24,282 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d2bddd611f30487d8d7ca6bfaac43951, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731899781307 2024-11-18T03:16:24,289 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 39d4fbba02ae4621927fefbd1d783f51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731899781669 2024-11-18T03:16:24,289 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c6ea17b9b943158d6ec48cc4a164a0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731899781669 2024-11-18T03:16:24,290 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b467701c574cb3a01b7977412ef1bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731899782936 2024-11-18T03:16:24,292 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0586d60dc9224a86930d68810eea8bcb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731899782936 2024-11-18T03:16:24,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T03:16:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:24,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T03:16:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:24,342 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:24,343 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/66f5a4aac69448359cd0250fdd25c880 is 50, key is test_row_0/B:col10/1731899782936/Put/seqid=0 2024-11-18T03:16:24,343 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f7fb6151a51241f09bf6357b8c213498_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899783505/Put/seqid=0 2024-11-18T03:16:24,358 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118f43adc1af81e465da22c2576bbb6810d_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:24,360 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118f43adc1af81e465da22c2576bbb6810d_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:24,360 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f43adc1af81e465da22c2576bbb6810d_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:24,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742205_1381 (size=12104) 2024-11-18T03:16:24,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742206_1382 (size=12154) 2024-11-18T03:16:24,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742207_1383 (size=4469) 2024-11-18T03:16:24,485 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#325 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:24,485 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/d81729eb77794c6ba8d0046147bcbef3 is 175, key is test_row_0/A:col10/1731899782936/Put/seqid=0 2024-11-18T03:16:24,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742208_1384 (size=31058) 2024-11-18T03:16:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T03:16:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:24,806 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/66f5a4aac69448359cd0250fdd25c880 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/66f5a4aac69448359cd0250fdd25c880 2024-11-18T03:16:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899844789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899844789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899844800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899844802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899844808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,827 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into 66f5a4aac69448359cd0250fdd25c880(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:24,827 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:24,827 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=13, startTime=1731899784274; duration=0sec 2024-11-18T03:16:24,827 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:24,827 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:24,828 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:24,831 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:24,831 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:24,831 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:24,831 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d86f852969f4fc79eedc756f424b532, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8348b4a1c0a1426a9a159913690c9451, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/4f3ec6359f9f472a9d331c513ad97b40] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=35.2 K 2024-11-18T03:16:24,831 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d86f852969f4fc79eedc756f424b532, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731899781307 2024-11-18T03:16:24,832 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8348b4a1c0a1426a9a159913690c9451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731899781669 2024-11-18T03:16:24,833 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3ec6359f9f472a9d331c513ad97b40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731899782936 2024-11-18T03:16:24,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:24,856 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f7fb6151a51241f09bf6357b8c213498_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f7fb6151a51241f09bf6357b8c213498_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:24,859 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:24,859 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/9fbaf57efbf1438ca3e2580a810c5b78 is 50, key is test_row_0/C:col10/1731899782936/Put/seqid=0 2024-11-18T03:16:24,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/29393b11b63e4fbfaedbabd788758719, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:24,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/29393b11b63e4fbfaedbabd788758719 is 175, key is test_row_0/A:col10/1731899783505/Put/seqid=0 2024-11-18T03:16:24,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742209_1385 (size=12104) 2024-11-18T03:16:24,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742210_1386 (size=30955) 2024-11-18T03:16:24,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899844913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899844915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,947 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/9fbaf57efbf1438ca3e2580a810c5b78 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9fbaf57efbf1438ca3e2580a810c5b78 2024-11-18T03:16:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899844931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899844935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899844932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:24,981 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into 9fbaf57efbf1438ca3e2580a810c5b78(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:24,981 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:24,981 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=13, startTime=1731899784274; duration=0sec 2024-11-18T03:16:24,981 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:24,982 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:24,987 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/d81729eb77794c6ba8d0046147bcbef3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d81729eb77794c6ba8d0046147bcbef3 2024-11-18T03:16:25,010 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into d81729eb77794c6ba8d0046147bcbef3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:25,010 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:25,010 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=13, startTime=1731899784274; duration=0sec 2024-11-18T03:16:25,010 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:25,011 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:25,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899845141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899845152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899845171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899845171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899845173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/29393b11b63e4fbfaedbabd788758719 2024-11-18T03:16:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/4194d27677ee46c29649ebe552fe8282 is 50, key is test_row_0/B:col10/1731899783505/Put/seqid=0 2024-11-18T03:16:25,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742211_1387 (size=12001) 2024-11-18T03:16:25,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899845487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899845492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899845513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899845522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899845523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T03:16:25,794 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/4194d27677ee46c29649ebe552fe8282 2024-11-18T03:16:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/d53113d8513943d0bbb2b4358476e0d6 is 50, key is test_row_0/C:col10/1731899783505/Put/seqid=0 2024-11-18T03:16:25,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742212_1388 (size=12001) 2024-11-18T03:16:25,857 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/d53113d8513943d0bbb2b4358476e0d6 2024-11-18T03:16:25,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/29393b11b63e4fbfaedbabd788758719 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/29393b11b63e4fbfaedbabd788758719 2024-11-18T03:16:25,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/29393b11b63e4fbfaedbabd788758719, entries=150, sequenceid=82, filesize=30.2 K 2024-11-18T03:16:25,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/4194d27677ee46c29649ebe552fe8282 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/4194d27677ee46c29649ebe552fe8282 2024-11-18T03:16:25,922 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/4194d27677ee46c29649ebe552fe8282, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T03:16:25,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/d53113d8513943d0bbb2b4358476e0d6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/d53113d8513943d0bbb2b4358476e0d6 2024-11-18T03:16:25,954 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/d53113d8513943d0bbb2b4358476e0d6, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T03:16:25,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 70fa1092abe5ed71c7b0809f585c5469 in 1665ms, sequenceid=82, compaction requested=false 2024-11-18T03:16:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-18T03:16:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-18T03:16:25,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-18T03:16:25,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4890 sec 2024-11-18T03:16:25,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.5040 sec 2024-11-18T03:16:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:26,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:16:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:26,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111853d787bda1ac49a7b9f8fa81ed29b64b_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899786107/Put/seqid=0 2024-11-18T03:16:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899846197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742213_1389 (size=14594) 2024-11-18T03:16:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899846197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899846222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899846225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899846232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899846341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899846344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899846351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899846352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899846362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899846564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899846564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899846571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899846572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899846593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,632 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:26,655 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111853d787bda1ac49a7b9f8fa81ed29b64b_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111853d787bda1ac49a7b9f8fa81ed29b64b_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:26,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/1aba114bbf4848689523f69f8cbb3b60, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:26,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/1aba114bbf4848689523f69f8cbb3b60 is 175, key is test_row_0/A:col10/1731899786107/Put/seqid=0 2024-11-18T03:16:26,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742214_1390 (size=39549) 2024-11-18T03:16:26,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/1aba114bbf4848689523f69f8cbb3b60 2024-11-18T03:16:26,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/0ba64ec17b524219a02074cf11816974 is 50, key is test_row_0/B:col10/1731899786107/Put/seqid=0 2024-11-18T03:16:26,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742215_1391 (size=12001) 2024-11-18T03:16:26,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/0ba64ec17b524219a02074cf11816974 2024-11-18T03:16:26,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/74a62601c20c44098283113122873ea9 is 50, key is test_row_0/C:col10/1731899786107/Put/seqid=0 2024-11-18T03:16:26,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742216_1392 (size=12001) 2024-11-18T03:16:26,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/74a62601c20c44098283113122873ea9 2024-11-18T03:16:26,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/1aba114bbf4848689523f69f8cbb3b60 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1aba114bbf4848689523f69f8cbb3b60 2024-11-18T03:16:26,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1aba114bbf4848689523f69f8cbb3b60, entries=200, sequenceid=102, filesize=38.6 K 2024-11-18T03:16:26,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/0ba64ec17b524219a02074cf11816974 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0ba64ec17b524219a02074cf11816974 2024-11-18T03:16:26,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0ba64ec17b524219a02074cf11816974, entries=150, sequenceid=102, filesize=11.7 K 2024-11-18T03:16:26,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/74a62601c20c44098283113122873ea9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74a62601c20c44098283113122873ea9 2024-11-18T03:16:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899846892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899846892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899846892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899846892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:26,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74a62601c20c44098283113122873ea9, entries=150, sequenceid=102, filesize=11.7 K 2024-11-18T03:16:26,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 70fa1092abe5ed71c7b0809f585c5469 in 820ms, sequenceid=102, compaction requested=true 2024-11-18T03:16:26,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:26,929 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:26,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:26,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:26,931 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:26,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:26,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:26,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:26,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:26,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:16:26,934 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:26,934 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:26,934 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:26,934 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d81729eb77794c6ba8d0046147bcbef3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/29393b11b63e4fbfaedbabd788758719, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1aba114bbf4848689523f69f8cbb3b60] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=99.2 K 2024-11-18T03:16:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:26,934 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:26,934 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d81729eb77794c6ba8d0046147bcbef3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/29393b11b63e4fbfaedbabd788758719, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1aba114bbf4848689523f69f8cbb3b60] 2024-11-18T03:16:26,935 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:26,935 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:26,935 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d81729eb77794c6ba8d0046147bcbef3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731899782936 2024-11-18T03:16:26,935 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:26,935 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/66f5a4aac69448359cd0250fdd25c880, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/4194d27677ee46c29649ebe552fe8282, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0ba64ec17b524219a02074cf11816974] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=35.3 K 2024-11-18T03:16:26,935 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29393b11b63e4fbfaedbabd788758719, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731899783474 2024-11-18T03:16:26,935 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f5a4aac69448359cd0250fdd25c880, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731899782936 2024-11-18T03:16:26,936 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aba114bbf4848689523f69f8cbb3b60, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899786040 2024-11-18T03:16:26,936 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 4194d27677ee46c29649ebe552fe8282, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731899783474 2024-11-18T03:16:26,937 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ba64ec17b524219a02074cf11816974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899786087 2024-11-18T03:16:26,950 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:26,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118088a7dc83ead444085aa57395e3a90f9_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899786927/Put/seqid=0 2024-11-18T03:16:26,954 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#335 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:26,954 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c7f83820fb034d3c92dcc21766c65589 is 50, key is test_row_0/B:col10/1731899786107/Put/seqid=0 2024-11-18T03:16:26,955 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411182efeb4bfc9f647b6989112a2800dfb83_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:26,962 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411182efeb4bfc9f647b6989112a2800dfb83_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:26,962 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182efeb4bfc9f647b6989112a2800dfb83_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742217_1393 (size=12207) 2024-11-18T03:16:27,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742218_1394 (size=14594) 2024-11-18T03:16:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742219_1395 (size=4469) 2024-11-18T03:16:27,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899847207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899847336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:27,404 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c7f83820fb034d3c92dcc21766c65589 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c7f83820fb034d3c92dcc21766c65589 2024-11-18T03:16:27,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:27,422 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into c7f83820fb034d3c92dcc21766c65589(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:27,422 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#334 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:27,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:27,422 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=13, startTime=1731899786930; duration=0sec 2024-11-18T03:16:27,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:27,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:27,422 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:27,423 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/8ae818cfa3f54da9b2c4bd2702c72bfa is 175, key is test_row_0/A:col10/1731899786107/Put/seqid=0 2024-11-18T03:16:27,431 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:27,431 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:27,431 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:27,432 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9fbaf57efbf1438ca3e2580a810c5b78, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/d53113d8513943d0bbb2b4358476e0d6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74a62601c20c44098283113122873ea9] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=35.3 K 2024-11-18T03:16:27,439 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbaf57efbf1438ca3e2580a810c5b78, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731899782936 2024-11-18T03:16:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899847411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899847411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899847412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899847423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,444 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d53113d8513943d0bbb2b4358476e0d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731899783474 2024-11-18T03:16:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:27,445 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118088a7dc83ead444085aa57395e3a90f9_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118088a7dc83ead444085aa57395e3a90f9_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:27,446 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a62601c20c44098283113122873ea9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899786087 2024-11-18T03:16:27,447 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/993ffe2619914ad7bbe6a6f77c7fb65c, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:27,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/993ffe2619914ad7bbe6a6f77c7fb65c is 175, key is test_row_0/A:col10/1731899786927/Put/seqid=0 2024-11-18T03:16:27,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742220_1396 (size=31161) 2024-11-18T03:16:27,486 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:27,486 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/8ae818cfa3f54da9b2c4bd2702c72bfa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/8ae818cfa3f54da9b2c4bd2702c72bfa 2024-11-18T03:16:27,486 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/c60247e5e2d14131b9f8c96c0982bea6 is 50, key is test_row_0/C:col10/1731899786107/Put/seqid=0 2024-11-18T03:16:27,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742221_1397 (size=39549) 2024-11-18T03:16:27,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/993ffe2619914ad7bbe6a6f77c7fb65c 2024-11-18T03:16:27,520 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into 8ae818cfa3f54da9b2c4bd2702c72bfa(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:27,520 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:27,520 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=13, startTime=1731899786929; duration=0sec 2024-11-18T03:16:27,520 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:27,520 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742222_1398 (size=12207) 2024-11-18T03:16:27,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/23920e8d469a450aa0575ea4720df547 is 50, key is test_row_0/B:col10/1731899786927/Put/seqid=0 2024-11-18T03:16:27,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899847551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:27,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742223_1399 (size=12001) 2024-11-18T03:16:27,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/23920e8d469a450aa0575ea4720df547 2024-11-18T03:16:27,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/9eb2febcb742408a9d8f6290c2d7da31 is 50, key is test_row_0/C:col10/1731899786927/Put/seqid=0 2024-11-18T03:16:27,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742224_1400 (size=12001) 2024-11-18T03:16:27,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/9eb2febcb742408a9d8f6290c2d7da31 2024-11-18T03:16:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T03:16:27,642 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:27,643 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-18T03:16:27,645 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:27,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:27,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T03:16:27,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/993ffe2619914ad7bbe6a6f77c7fb65c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/993ffe2619914ad7bbe6a6f77c7fb65c 2024-11-18T03:16:27,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/993ffe2619914ad7bbe6a6f77c7fb65c, entries=200, sequenceid=119, filesize=38.6 K 2024-11-18T03:16:27,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/23920e8d469a450aa0575ea4720df547 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/23920e8d469a450aa0575ea4720df547 2024-11-18T03:16:27,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/23920e8d469a450aa0575ea4720df547, entries=150, sequenceid=119, filesize=11.7 K 2024-11-18T03:16:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/9eb2febcb742408a9d8f6290c2d7da31 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9eb2febcb742408a9d8f6290c2d7da31 2024-11-18T03:16:27,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9eb2febcb742408a9d8f6290c2d7da31, entries=150, sequenceid=119, filesize=11.7 K 2024-11-18T03:16:27,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 70fa1092abe5ed71c7b0809f585c5469 in 774ms, sequenceid=119, compaction requested=false 2024-11-18T03:16:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T03:16:27,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T03:16:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:27,805 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:16:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:27,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f9831c03685c42ff9bbb991ae653056c_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899787198/Put/seqid=0 2024-11-18T03:16:27,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742225_1401 (size=12254) 2024-11-18T03:16:27,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T03:16:27,975 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/c60247e5e2d14131b9f8c96c0982bea6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c60247e5e2d14131b9f8c96c0982bea6 2024-11-18T03:16:28,000 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into c60247e5e2d14131b9f8c96c0982bea6(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:28,000 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:28,000 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=13, startTime=1731899786931; duration=0sec 2024-11-18T03:16:28,001 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:28,001 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:28,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T03:16:28,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f9831c03685c42ff9bbb991ae653056c_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9831c03685c42ff9bbb991ae653056c_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06bb5b847fa2425dae52392cd4ebf096, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:28,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06bb5b847fa2425dae52392cd4ebf096 is 175, key is test_row_0/A:col10/1731899787198/Put/seqid=0 2024-11-18T03:16:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899848293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742226_1402 (size=31055) 2024-11-18T03:16:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899848424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899848451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,472 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in 70fa1092abe5ed71c7b0809f585c5469 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899848451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899848452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,473 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 70fa1092abe5ed71c7b0809f585c5469 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,474 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 70fa1092abe5ed71c7b0809f585c5469 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899848463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,507 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 70fa1092abe5ed71c7b0809f585c5469 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899848631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,744 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06bb5b847fa2425dae52392cd4ebf096 2024-11-18T03:16:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T03:16:28,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f155f2b7908944e49771203846ed090f is 50, key is test_row_0/B:col10/1731899787198/Put/seqid=0 2024-11-18T03:16:28,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742227_1403 (size=12101) 2024-11-18T03:16:28,853 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f155f2b7908944e49771203846ed090f 2024-11-18T03:16:28,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/e269469acea24a6f8d017adc50baebf9 is 50, key is test_row_0/C:col10/1731899787198/Put/seqid=0 2024-11-18T03:16:28,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742228_1404 (size=12101) 2024-11-18T03:16:28,950 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/e269469acea24a6f8d017adc50baebf9 2024-11-18T03:16:28,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899848952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:28,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06bb5b847fa2425dae52392cd4ebf096 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06bb5b847fa2425dae52392cd4ebf096 2024-11-18T03:16:29,002 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06bb5b847fa2425dae52392cd4ebf096, entries=150, sequenceid=136, filesize=30.3 K 2024-11-18T03:16:29,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f155f2b7908944e49771203846ed090f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f155f2b7908944e49771203846ed090f 2024-11-18T03:16:29,028 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f155f2b7908944e49771203846ed090f, entries=150, sequenceid=136, filesize=11.8 K 2024-11-18T03:16:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/e269469acea24a6f8d017adc50baebf9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/e269469acea24a6f8d017adc50baebf9 2024-11-18T03:16:29,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/e269469acea24a6f8d017adc50baebf9, entries=150, sequenceid=136, filesize=11.8 K 2024-11-18T03:16:29,062 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 70fa1092abe5ed71c7b0809f585c5469 in 1257ms, sequenceid=136, compaction requested=true 2024-11-18T03:16:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-18T03:16:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-18T03:16:29,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-18T03:16:29,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4250 sec 2024-11-18T03:16:29,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4320 sec 2024-11-18T03:16:29,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:16:29,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:29,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:29,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:29,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:29,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:29,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:29,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f9df5585046942a6aa1561aa2a017ec1_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899788247/Put/seqid=0 2024-11-18T03:16:29,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742229_1405 (size=14794) 2024-11-18T03:16:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T03:16:29,813 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:29,813 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-18T03:16:29,817 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T03:16:29,818 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:29,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:29,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899849833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T03:16:29,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T03:16:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:29,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:29,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:29,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:29,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:29,987 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:29,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899849962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:29,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f9df5585046942a6aa1561aa2a017ec1_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9df5585046942a6aa1561aa2a017ec1_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:29,993 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/9b9677c13b5d4a82a3c283820ed7ea3e, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:29,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/9b9677c13b5d4a82a3c283820ed7ea3e is 175, key is test_row_0/A:col10/1731899788247/Put/seqid=0 2024-11-18T03:16:30,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742230_1406 (size=39749) 2024-11-18T03:16:30,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/9b9677c13b5d4a82a3c283820ed7ea3e 2024-11-18T03:16:30,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c685a4eee2594848a17fc70dc44b61e4 is 50, key is test_row_0/B:col10/1731899788247/Put/seqid=0 2024-11-18T03:16:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T03:16:30,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T03:16:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:30,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:30,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742231_1407 (size=12151) 2024-11-18T03:16:30,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c685a4eee2594848a17fc70dc44b61e4 2024-11-18T03:16:30,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/c2a5a5f7e09847769c19dd85644eb948 is 50, key is test_row_0/C:col10/1731899788247/Put/seqid=0 2024-11-18T03:16:30,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742232_1408 (size=12151) 2024-11-18T03:16:30,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/c2a5a5f7e09847769c19dd85644eb948 2024-11-18T03:16:30,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/9b9677c13b5d4a82a3c283820ed7ea3e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/9b9677c13b5d4a82a3c283820ed7ea3e 2024-11-18T03:16:30,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899850214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/9b9677c13b5d4a82a3c283820ed7ea3e, entries=200, sequenceid=155, filesize=38.8 K 2024-11-18T03:16:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c685a4eee2594848a17fc70dc44b61e4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c685a4eee2594848a17fc70dc44b61e4 2024-11-18T03:16:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c685a4eee2594848a17fc70dc44b61e4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T03:16:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/c2a5a5f7e09847769c19dd85644eb948 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c2a5a5f7e09847769c19dd85644eb948 2024-11-18T03:16:30,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c2a5a5f7e09847769c19dd85644eb948, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T03:16:30,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 70fa1092abe5ed71c7b0809f585c5469 in 770ms, sequenceid=155, compaction requested=true 2024-11-18T03:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:30,273 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:16:30,274 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:16:30,274 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:30,274 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:30,275 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/8ae818cfa3f54da9b2c4bd2702c72bfa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/993ffe2619914ad7bbe6a6f77c7fb65c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06bb5b847fa2425dae52392cd4ebf096, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/9b9677c13b5d4a82a3c283820ed7ea3e] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=138.2 K 2024-11-18T03:16:30,276 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:16:30,276 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:30,276 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/8ae818cfa3f54da9b2c4bd2702c72bfa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/993ffe2619914ad7bbe6a6f77c7fb65c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06bb5b847fa2425dae52392cd4ebf096, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/9b9677c13b5d4a82a3c283820ed7ea3e] 2024-11-18T03:16:30,284 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae818cfa3f54da9b2c4bd2702c72bfa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899786087 2024-11-18T03:16:30,288 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:16:30,288 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:30,288 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:30,288 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c7f83820fb034d3c92dcc21766c65589, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/23920e8d469a450aa0575ea4720df547, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f155f2b7908944e49771203846ed090f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c685a4eee2594848a17fc70dc44b61e4] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=47.3 K 2024-11-18T03:16:30,291 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 993ffe2619914ad7bbe6a6f77c7fb65c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731899786212 2024-11-18T03:16:30,291 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7f83820fb034d3c92dcc21766c65589, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899786087 2024-11-18T03:16:30,292 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 06bb5b847fa2425dae52392cd4ebf096, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731899787039 2024-11-18T03:16:30,292 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23920e8d469a450aa0575ea4720df547, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731899786212 2024-11-18T03:16:30,295 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b9677c13b5d4a82a3c283820ed7ea3e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731899788067 2024-11-18T03:16:30,295 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f155f2b7908944e49771203846ed090f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731899787039 2024-11-18T03:16:30,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T03:16:30,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:30,296 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T03:16:30,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:30,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:30,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:30,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:30,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:30,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:30,300 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c685a4eee2594848a17fc70dc44b61e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731899788247 2024-11-18T03:16:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184e5161d97f3c46958f08df2a1716b298_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899789800/Put/seqid=0 2024-11-18T03:16:30,348 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:30,348 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/b660c397e8614301aca6dcb2b9a2e12a is 50, key is test_row_0/B:col10/1731899788247/Put/seqid=0 2024-11-18T03:16:30,358 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:30,408 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411183797428583f0456cb3a82fb58bbc027e_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:30,411 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411183797428583f0456cb3a82fb58bbc027e_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:30,411 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183797428583f0456cb3a82fb58bbc027e_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:30,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742233_1409 (size=12304) 2024-11-18T03:16:30,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T03:16:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742234_1410 (size=12493) 2024-11-18T03:16:30,470 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184e5161d97f3c46958f08df2a1716b298_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e5161d97f3c46958f08df2a1716b298_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:30,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/08cd1765a51d4c03bc4f77e65507e1d8, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:30,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/08cd1765a51d4c03bc4f77e65507e1d8 is 175, key is test_row_0/A:col10/1731899789800/Put/seqid=0 2024-11-18T03:16:30,484 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/b660c397e8614301aca6dcb2b9a2e12a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/b660c397e8614301aca6dcb2b9a2e12a 2024-11-18T03:16:30,515 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into b660c397e8614301aca6dcb2b9a2e12a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:30,515 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:30,515 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=12, startTime=1731899790273; duration=0sec 2024-11-18T03:16:30,515 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:30,515 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:30,515 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:16:30,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742235_1411 (size=4469) 2024-11-18T03:16:30,521 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:16:30,521 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:30,521 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:30,521 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c60247e5e2d14131b9f8c96c0982bea6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9eb2febcb742408a9d8f6290c2d7da31, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/e269469acea24a6f8d017adc50baebf9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c2a5a5f7e09847769c19dd85644eb948] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=47.3 K 2024-11-18T03:16:30,522 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60247e5e2d14131b9f8c96c0982bea6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731899786087 2024-11-18T03:16:30,522 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eb2febcb742408a9d8f6290c2d7da31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731899786212 2024-11-18T03:16:30,523 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e269469acea24a6f8d017adc50baebf9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731899787039 2024-11-18T03:16:30,523 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a5a5f7e09847769c19dd85644eb948, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731899788247 2024-11-18T03:16:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:30,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742236_1412 (size=31105) 2024-11-18T03:16:30,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/08cd1765a51d4c03bc4f77e65507e1d8 2024-11-18T03:16:30,562 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#348 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:30,563 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/6f1ced17295b4181baf67397f81527d5 is 50, key is test_row_0/C:col10/1731899788247/Put/seqid=0 2024-11-18T03:16:30,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/d4b09d3032a8415a91f11a8cf10a497a is 50, key is test_row_0/B:col10/1731899789800/Put/seqid=0 2024-11-18T03:16:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742237_1413 (size=12493) 2024-11-18T03:16:30,626 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/6f1ced17295b4181baf67397f81527d5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6f1ced17295b4181baf67397f81527d5 2024-11-18T03:16:30,633 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into 6f1ced17295b4181baf67397f81527d5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:30,633 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:30,633 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=12, startTime=1731899790273; duration=0sec 2024-11-18T03:16:30,634 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:30,634 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:30,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742238_1414 (size=12151) 2024-11-18T03:16:30,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899850652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899850657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899850685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899850687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899850700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899850811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899850811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899850841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899850842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899850853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:30,922 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#347 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:30,922 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/31836cb9aac242548c6236ec84687f2e is 175, key is test_row_0/A:col10/1731899788247/Put/seqid=0 2024-11-18T03:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T03:16:30,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742239_1415 (size=31447) 2024-11-18T03:16:30,965 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/31836cb9aac242548c6236ec84687f2e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/31836cb9aac242548c6236ec84687f2e 2024-11-18T03:16:30,979 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into 31836cb9aac242548c6236ec84687f2e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:30,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:30,979 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=12, startTime=1731899790273; duration=0sec 2024-11-18T03:16:30,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:30,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899851044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899851051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899851071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899851081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,115 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/d4b09d3032a8415a91f11a8cf10a497a 2024-11-18T03:16:31,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899851092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/0ff6f04d971c4e1cbfadc995b5045974 is 50, key is test_row_0/C:col10/1731899789800/Put/seqid=0 2024-11-18T03:16:31,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742240_1416 (size=12151) 2024-11-18T03:16:31,192 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/0ff6f04d971c4e1cbfadc995b5045974 2024-11-18T03:16:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/08cd1765a51d4c03bc4f77e65507e1d8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/08cd1765a51d4c03bc4f77e65507e1d8 2024-11-18T03:16:31,216 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/08cd1765a51d4c03bc4f77e65507e1d8, entries=150, sequenceid=169, filesize=30.4 K 2024-11-18T03:16:31,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/d4b09d3032a8415a91f11a8cf10a497a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d4b09d3032a8415a91f11a8cf10a497a 2024-11-18T03:16:31,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d4b09d3032a8415a91f11a8cf10a497a, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T03:16:31,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/0ff6f04d971c4e1cbfadc995b5045974 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/0ff6f04d971c4e1cbfadc995b5045974 2024-11-18T03:16:31,230 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/0ff6f04d971c4e1cbfadc995b5045974, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T03:16:31,231 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 70fa1092abe5ed71c7b0809f585c5469 in 935ms, sequenceid=169, compaction requested=false 2024-11-18T03:16:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-18T03:16:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-18T03:16:31,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-18T03:16:31,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-18T03:16:31,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4240 sec 2024-11-18T03:16:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:31,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T03:16:31,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:31,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:31,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:31,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:31,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:31,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:31,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118937759576f2f4b95b098369f67655bdb_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899790697/Put/seqid=0 2024-11-18T03:16:31,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742241_1417 (size=14794) 2024-11-18T03:16:31,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899851508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899851508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899851521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899851540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899851550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899851651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899851661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899851663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899851673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899851691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899851871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:31,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899851884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899851884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118937759576f2f4b95b098369f67655bdb_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118937759576f2f4b95b098369f67655bdb_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:31,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/7b06ba4e1cfd488ab2e5dd7b221d17f1, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:31,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/7b06ba4e1cfd488ab2e5dd7b221d17f1 is 175, key is test_row_0/A:col10/1731899790697/Put/seqid=0 2024-11-18T03:16:31,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899851884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899851911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:31,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742242_1418 (size=39749) 2024-11-18T03:16:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T03:16:31,971 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:31,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-18T03:16:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:31,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:31,977 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:31,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:32,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899852191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899852201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899852201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899852212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899852231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:32,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:32,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:32,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/7b06ba4e1cfd488ab2e5dd7b221d17f1 2024-11-18T03:16:32,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/06b0995226134166afa684d3f48cad1b is 50, key is test_row_0/B:col10/1731899790697/Put/seqid=0 2024-11-18T03:16:32,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742243_1419 (size=12151) 2024-11-18T03:16:32,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/06b0995226134166afa684d3f48cad1b 2024-11-18T03:16:32,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/74f7885647c940cd9a6943627d552f1d is 50, key is test_row_0/C:col10/1731899790697/Put/seqid=0 2024-11-18T03:16:32,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742244_1420 (size=12151) 2024-11-18T03:16:32,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/74f7885647c940cd9a6943627d552f1d 2024-11-18T03:16:32,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/7b06ba4e1cfd488ab2e5dd7b221d17f1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/7b06ba4e1cfd488ab2e5dd7b221d17f1 2024-11-18T03:16:32,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/7b06ba4e1cfd488ab2e5dd7b221d17f1, entries=200, sequenceid=192, filesize=38.8 K 2024-11-18T03:16:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/06b0995226134166afa684d3f48cad1b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/06b0995226134166afa684d3f48cad1b 2024-11-18T03:16:32,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/06b0995226134166afa684d3f48cad1b, entries=150, sequenceid=192, filesize=11.9 K 2024-11-18T03:16:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:32,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/74f7885647c940cd9a6943627d552f1d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74f7885647c940cd9a6943627d552f1d 2024-11-18T03:16:32,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:32,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74f7885647c940cd9a6943627d552f1d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-18T03:16:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 70fa1092abe5ed71c7b0809f585c5469 in 1243ms, sequenceid=192, compaction requested=true 2024-11-18T03:16:32,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:32,644 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:32,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:32,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:32,644 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:32,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:32,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:32,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:32,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:32,653 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:32,653 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:32,653 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,653 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/31836cb9aac242548c6236ec84687f2e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/08cd1765a51d4c03bc4f77e65507e1d8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/7b06ba4e1cfd488ab2e5dd7b221d17f1] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=99.9 K 2024-11-18T03:16:32,653 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,653 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/31836cb9aac242548c6236ec84687f2e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/08cd1765a51d4c03bc4f77e65507e1d8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/7b06ba4e1cfd488ab2e5dd7b221d17f1] 2024-11-18T03:16:32,653 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:32,653 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:32,653 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,653 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/b660c397e8614301aca6dcb2b9a2e12a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d4b09d3032a8415a91f11a8cf10a497a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/06b0995226134166afa684d3f48cad1b] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=35.9 K 2024-11-18T03:16:32,664 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b660c397e8614301aca6dcb2b9a2e12a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731899788247 2024-11-18T03:16:32,664 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31836cb9aac242548c6236ec84687f2e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731899788247 2024-11-18T03:16:32,672 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b09d3032a8415a91f11a8cf10a497a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731899789761 2024-11-18T03:16:32,672 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08cd1765a51d4c03bc4f77e65507e1d8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731899789761 2024-11-18T03:16:32,674 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b06ba4e1cfd488ab2e5dd7b221d17f1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731899790653 2024-11-18T03:16:32,674 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b0995226134166afa684d3f48cad1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731899790653 2024-11-18T03:16:32,714 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:32,724 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:32,725 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f7c7dcc95ae6450fbde86332ab6d868b is 50, key is test_row_0/B:col10/1731899790697/Put/seqid=0 2024-11-18T03:16:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:32,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T03:16:32,729 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111853d7cd11f5f54caa9a891515b9c263e9_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:32,731 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111853d7cd11f5f54caa9a891515b9c263e9_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:32,732 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111853d7cd11f5f54caa9a891515b9c263e9_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:32,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742245_1421 (size=12595) 2024-11-18T03:16:32,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111828669a8f00ad4c0da88c590be95d1223_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899791506/Put/seqid=0 2024-11-18T03:16:32,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742246_1422 (size=4469) 2024-11-18T03:16:32,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:32,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:32,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742247_1423 (size=14794) 2024-11-18T03:16:32,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:32,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899852824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899852827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899852842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111828669a8f00ad4c0da88c590be95d1223_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828669a8f00ad4c0da88c590be95d1223_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:32,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/5ffa5b6ae308484397c2e0fcec68468b, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:32,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/5ffa5b6ae308484397c2e0fcec68468b is 175, key is test_row_0/A:col10/1731899791506/Put/seqid=0 2024-11-18T03:16:32,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899852868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899852868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742248_1424 (size=39749) 2024-11-18T03:16:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:32,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:32,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:32,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:32,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899852972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899852972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899852976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:32,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:32,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899853016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899853022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:33,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:33,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,196 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f7c7dcc95ae6450fbde86332ab6d868b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f7c7dcc95ae6450fbde86332ab6d868b 2024-11-18T03:16:33,199 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#354 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:33,199 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/ce794658b0b049d29dfc85ef611dd1e2 is 175, key is test_row_0/A:col10/1731899790697/Put/seqid=0 2024-11-18T03:16:33,204 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into f7c7dcc95ae6450fbde86332ab6d868b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:33,204 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:33,204 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=13, startTime=1731899792644; duration=0sec 2024-11-18T03:16:33,204 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:33,204 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:33,204 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:33,208 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:33,209 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:33,209 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,209 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6f1ced17295b4181baf67397f81527d5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/0ff6f04d971c4e1cbfadc995b5045974, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74f7885647c940cd9a6943627d552f1d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=35.9 K 2024-11-18T03:16:33,209 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1ced17295b4181baf67397f81527d5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731899788247 2024-11-18T03:16:33,210 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff6f04d971c4e1cbfadc995b5045974, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731899789761 2024-11-18T03:16:33,210 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f7885647c940cd9a6943627d552f1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731899790653 2024-11-18T03:16:33,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899853203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899853203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899853203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,221 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:33,222 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/a90c1787c9804b38a47341792e8f9d49 is 50, key is test_row_0/C:col10/1731899790697/Put/seqid=0 2024-11-18T03:16:33,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742249_1425 (size=31549) 2024-11-18T03:16:33,234 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/ce794658b0b049d29dfc85ef611dd1e2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ce794658b0b049d29dfc85ef611dd1e2 2024-11-18T03:16:33,246 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into ce794658b0b049d29dfc85ef611dd1e2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:33,246 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:33,246 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=13, startTime=1731899792644; duration=0sec 2024-11-18T03:16:33,246 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:33,246 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899853239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899853239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742250_1426 (size=12595) 2024-11-18T03:16:33,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/5ffa5b6ae308484397c2e0fcec68468b 2024-11-18T03:16:33,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/42e074495a0547c5a14e50572920b9e9 is 50, key is test_row_0/B:col10/1731899791506/Put/seqid=0 2024-11-18T03:16:33,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742251_1427 (size=12151) 2024-11-18T03:16:33,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/42e074495a0547c5a14e50572920b9e9 2024-11-18T03:16:33,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3d5520e71de74b3687568d501d20fb95 is 50, key is test_row_0/C:col10/1731899791506/Put/seqid=0 2024-11-18T03:16:33,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:33,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:33,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742252_1428 (size=12151) 2024-11-18T03:16:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899853532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899853532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899853532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899853565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899853567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:33,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:33,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:33,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,721 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/a90c1787c9804b38a47341792e8f9d49 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a90c1787c9804b38a47341792e8f9d49 2024-11-18T03:16:33,752 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into a90c1787c9804b38a47341792e8f9d49(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:33,752 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:33,752 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=13, startTime=1731899792645; duration=0sec 2024-11-18T03:16:33,753 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:33,753 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:33,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:33,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:33,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3d5520e71de74b3687568d501d20fb95 2024-11-18T03:16:33,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/5ffa5b6ae308484397c2e0fcec68468b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/5ffa5b6ae308484397c2e0fcec68468b 2024-11-18T03:16:33,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:33,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/5ffa5b6ae308484397c2e0fcec68468b, entries=200, sequenceid=206, filesize=38.8 K 2024-11-18T03:16:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/42e074495a0547c5a14e50572920b9e9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/42e074495a0547c5a14e50572920b9e9 2024-11-18T03:16:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:33,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/42e074495a0547c5a14e50572920b9e9, entries=150, sequenceid=206, filesize=11.9 K 2024-11-18T03:16:33,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3d5520e71de74b3687568d501d20fb95 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d5520e71de74b3687568d501d20fb95 2024-11-18T03:16:33,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d5520e71de74b3687568d501d20fb95, entries=150, sequenceid=206, filesize=11.9 K 2024-11-18T03:16:34,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 70fa1092abe5ed71c7b0809f585c5469 in 1273ms, sequenceid=206, compaction requested=false 2024-11-18T03:16:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:34,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T03:16:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:34,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:34,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:34,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:34,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189996f571d8f246989a8176c9e6bb4274_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899792856/Put/seqid=0 2024-11-18T03:16:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899854107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899854111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899854112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899854131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899854134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742253_1429 (size=14794) 2024-11-18T03:16:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899854243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899854243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899854262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899854262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899854276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899854461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899854461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899854471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899854481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899854491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,568 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:34,601 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189996f571d8f246989a8176c9e6bb4274_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189996f571d8f246989a8176c9e6bb4274_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:34,613 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/3aac6a02ae734420afc9c47b0153a4dd, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:34,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/3aac6a02ae734420afc9c47b0153a4dd is 175, key is test_row_0/A:col10/1731899792856/Put/seqid=0 2024-11-18T03:16:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742254_1430 (size=39749) 2024-11-18T03:16:34,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/3aac6a02ae734420afc9c47b0153a4dd 2024-11-18T03:16:34,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:34,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/704eeafc34344989a60e758f5673540a is 50, key is test_row_0/B:col10/1731899792856/Put/seqid=0 2024-11-18T03:16:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899854790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899854791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899854797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742255_1431 (size=12151) 2024-11-18T03:16:34,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899854803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/704eeafc34344989a60e758f5673540a 2024-11-18T03:16:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899854824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:34,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/7b1ba919316b4b428eab60adeb6ea2d3 is 50, key is test_row_0/C:col10/1731899792856/Put/seqid=0 2024-11-18T03:16:34,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742256_1432 (size=12151) 2024-11-18T03:16:34,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:34,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:35,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:35,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:35,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:35,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:35,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:35,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:35,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/7b1ba919316b4b428eab60adeb6ea2d3 2024-11-18T03:16:35,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899855311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899855312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:35,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899855312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:35,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/3aac6a02ae734420afc9c47b0153a4dd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/3aac6a02ae734420afc9c47b0153a4dd 2024-11-18T03:16:35,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/3aac6a02ae734420afc9c47b0153a4dd, entries=200, sequenceid=229, filesize=38.8 K 2024-11-18T03:16:35,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899855332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:35,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/704eeafc34344989a60e758f5673540a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/704eeafc34344989a60e758f5673540a 2024-11-18T03:16:35,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/704eeafc34344989a60e758f5673540a, entries=150, sequenceid=229, filesize=11.9 K 2024-11-18T03:16:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899855352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/7b1ba919316b4b428eab60adeb6ea2d3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/7b1ba919316b4b428eab60adeb6ea2d3 2024-11-18T03:16:35,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/7b1ba919316b4b428eab60adeb6ea2d3, entries=150, sequenceid=229, filesize=11.9 K 2024-11-18T03:16:35,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 70fa1092abe5ed71c7b0809f585c5469 in 1331ms, sequenceid=229, compaction requested=true 2024-11-18T03:16:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:35,408 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:35,408 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:35,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:35,409 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:35,409 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:35,409 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:35,409 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:35,409 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,409 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,409 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f7c7dcc95ae6450fbde86332ab6d868b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/42e074495a0547c5a14e50572920b9e9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/704eeafc34344989a60e758f5673540a] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=36.0 K 2024-11-18T03:16:35,409 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ce794658b0b049d29dfc85ef611dd1e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/5ffa5b6ae308484397c2e0fcec68468b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/3aac6a02ae734420afc9c47b0153a4dd] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=108.4 K 2024-11-18T03:16:35,409 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,409 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ce794658b0b049d29dfc85ef611dd1e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/5ffa5b6ae308484397c2e0fcec68468b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/3aac6a02ae734420afc9c47b0153a4dd] 2024-11-18T03:16:35,410 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f7c7dcc95ae6450fbde86332ab6d868b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731899790653 2024-11-18T03:16:35,410 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce794658b0b049d29dfc85ef611dd1e2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731899790653 2024-11-18T03:16:35,410 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 42e074495a0547c5a14e50572920b9e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731899791506 2024-11-18T03:16:35,410 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ffa5b6ae308484397c2e0fcec68468b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731899791434 2024-11-18T03:16:35,410 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 704eeafc34344989a60e758f5673540a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731899792837 2024-11-18T03:16:35,410 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aac6a02ae734420afc9c47b0153a4dd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731899792837 2024-11-18T03:16:35,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T03:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,430 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T03:16:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:35,444 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:35,445 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/e138c1389aee42838c0461ce69d7f8c2 is 50, key is test_row_0/B:col10/1731899792856/Put/seqid=0 2024-11-18T03:16:35,452 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:35,471 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411186118c12ca15d4724bdd26a66723e6f59_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:35,473 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411186118c12ca15d4724bdd26a66723e6f59_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:35,473 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186118c12ca15d4724bdd26a66723e6f59_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187dfa4f04d4734816bbc999e7d32b4c51_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899794098/Put/seqid=0 2024-11-18T03:16:35,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742257_1433 (size=12697) 2024-11-18T03:16:35,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742259_1435 (size=12304) 2024-11-18T03:16:35,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742258_1434 (size=4469) 2024-11-18T03:16:35,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:35,614 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#364 average throughput is 0.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:35,615 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/465d1b7909fc4c858a12d7c5d9f31d29 is 175, key is test_row_0/A:col10/1731899792856/Put/seqid=0 2024-11-18T03:16:35,616 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187dfa4f04d4734816bbc999e7d32b4c51_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187dfa4f04d4734816bbc999e7d32b4c51_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/54daea1dc15347dd9d3c9eae14fac69a, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:35,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/54daea1dc15347dd9d3c9eae14fac69a is 175, key is test_row_0/A:col10/1731899794098/Put/seqid=0 2024-11-18T03:16:35,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742260_1436 (size=31651) 2024-11-18T03:16:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742261_1437 (size=31105) 2024-11-18T03:16:35,917 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/e138c1389aee42838c0461ce69d7f8c2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/e138c1389aee42838c0461ce69d7f8c2 2024-11-18T03:16:35,928 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into e138c1389aee42838c0461ce69d7f8c2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:35,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:35,928 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=13, startTime=1731899795408; duration=0sec 2024-11-18T03:16:35,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:35,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:35,928 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:35,929 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:35,929 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:35,929 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:35,929 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a90c1787c9804b38a47341792e8f9d49, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d5520e71de74b3687568d501d20fb95, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/7b1ba919316b4b428eab60adeb6ea2d3] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=36.0 K 2024-11-18T03:16:35,929 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a90c1787c9804b38a47341792e8f9d49, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731899790653 2024-11-18T03:16:35,930 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d5520e71de74b3687568d501d20fb95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731899791506 2024-11-18T03:16:35,930 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b1ba919316b4b428eab60adeb6ea2d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731899792837 2024-11-18T03:16:35,938 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:35,939 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/a66fd744248d4b55ab8954b946878198 is 50, key is test_row_0/C:col10/1731899792856/Put/seqid=0 2024-11-18T03:16:35,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742262_1438 (size=12697) 2024-11-18T03:16:36,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/54daea1dc15347dd9d3c9eae14fac69a 2024-11-18T03:16:36,093 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/465d1b7909fc4c858a12d7c5d9f31d29 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/465d1b7909fc4c858a12d7c5d9f31d29 2024-11-18T03:16:36,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/841d74d31d274fab9cf966d6f8518708 is 50, key is test_row_0/B:col10/1731899794098/Put/seqid=0 2024-11-18T03:16:36,108 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into 465d1b7909fc4c858a12d7c5d9f31d29(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:36,109 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:36,109 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=13, startTime=1731899795408; duration=0sec 2024-11-18T03:16:36,109 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:36,109 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742263_1439 (size=12151) 2024-11-18T03:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:36,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/841d74d31d274fab9cf966d6f8518708 2024-11-18T03:16:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/b5b756f63c73495f861f1bd8ba448da5 is 50, key is test_row_0/C:col10/1731899794098/Put/seqid=0 2024-11-18T03:16:36,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742264_1440 (size=12151) 2024-11-18T03:16:36,256 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/b5b756f63c73495f861f1bd8ba448da5 2024-11-18T03:16:36,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/54daea1dc15347dd9d3c9eae14fac69a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/54daea1dc15347dd9d3c9eae14fac69a 2024-11-18T03:16:36,316 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/54daea1dc15347dd9d3c9eae14fac69a, entries=150, sequenceid=241, filesize=30.4 K 2024-11-18T03:16:36,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/841d74d31d274fab9cf966d6f8518708 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/841d74d31d274fab9cf966d6f8518708 2024-11-18T03:16:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/841d74d31d274fab9cf966d6f8518708, entries=150, sequenceid=241, filesize=11.9 K 2024-11-18T03:16:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/b5b756f63c73495f861f1bd8ba448da5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b5b756f63c73495f861f1bd8ba448da5 2024-11-18T03:16:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,406 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b5b756f63c73495f861f1bd8ba448da5, entries=150, sequenceid=241, filesize=11.9 K 2024-11-18T03:16:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,409 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 70fa1092abe5ed71c7b0809f585c5469 in 980ms, sequenceid=241, compaction requested=false 2024-11-18T03:16:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-18T03:16:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-18T03:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-18T03:16:36,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4370 sec 2024-11-18T03:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,419 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/a66fd744248d4b55ab8954b946878198 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a66fd744248d4b55ab8954b946878198 2024-11-18T03:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 4.4470 sec 2024-11-18T03:16:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,459 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into a66fd744248d4b55ab8954b946878198(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:36,459 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:36,459 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=13, startTime=1731899795408; duration=0sec 2024-11-18T03:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,460 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:36,460 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:36,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:16:36,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:36,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:36,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:36,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:36,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:36,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182359a29bf9d8456ba6118e01e4184fb2_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899796839/Put/seqid=0 2024-11-18T03:16:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742265_1441 (size=22268) 2024-11-18T03:16:36,952 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,956 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182359a29bf9d8456ba6118e01e4184fb2_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182359a29bf9d8456ba6118e01e4184fb2_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/254da17333c4431189f90fd6778e476f, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/254da17333c4431189f90fd6778e476f is 175, key is test_row_0/A:col10/1731899796839/Put/seqid=0 2024-11-18T03:16:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:36,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742266_1442 (size=65673) 2024-11-18T03:16:37,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731899796971, max timestamp=1731899797009Num uniques 859; 2024-11-18T03:16:37,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731899796971, max timestamp=1731899797009Num uniques 859; 2024-11-18T03:16:37,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-18T03:16:37,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-18T03:16:37,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731899796971, max timestamp=1731899797009Num uniques 859; 2024-11-18T03:16:37,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-18T03:16:37,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899857070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899857072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899857092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899857100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899857106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899857211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899857211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899857231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899857232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899857232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/254da17333c4431189f90fd6778e476f 2024-11-18T03:16:37,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/41e520b4524c4aa2af74e1fc5759f5e0 is 50, key is test_row_0/B:col10/1731899796839/Put/seqid=0 2024-11-18T03:16:37,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742267_1443 (size=12151) 2024-11-18T03:16:37,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/41e520b4524c4aa2af74e1fc5759f5e0 2024-11-18T03:16:37,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899857443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899857443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/8b151e51f77542c98a807baf70118b43 is 50, key is test_row_0/C:col10/1731899796839/Put/seqid=0 2024-11-18T03:16:37,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899857461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899857461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899857462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742268_1444 (size=12151) 2024-11-18T03:16:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899857782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899857782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899857802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899857802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899857802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:37,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/8b151e51f77542c98a807baf70118b43 2024-11-18T03:16:37,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/254da17333c4431189f90fd6778e476f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/254da17333c4431189f90fd6778e476f 2024-11-18T03:16:37,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/254da17333c4431189f90fd6778e476f, entries=350, sequenceid=254, filesize=64.1 K 2024-11-18T03:16:37,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/41e520b4524c4aa2af74e1fc5759f5e0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/41e520b4524c4aa2af74e1fc5759f5e0 2024-11-18T03:16:37,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/41e520b4524c4aa2af74e1fc5759f5e0, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T03:16:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/8b151e51f77542c98a807baf70118b43 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8b151e51f77542c98a807baf70118b43 2024-11-18T03:16:37,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8b151e51f77542c98a807baf70118b43, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T03:16:37,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 70fa1092abe5ed71c7b0809f585c5469 in 1096ms, sequenceid=254, compaction requested=true 2024-11-18T03:16:37,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:37,965 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:37,965 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:37,969 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128429 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:37,969 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:37,969 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:37,969 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/465d1b7909fc4c858a12d7c5d9f31d29, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/54daea1dc15347dd9d3c9eae14fac69a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/254da17333c4431189f90fd6778e476f] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=125.4 K 2024-11-18T03:16:37,969 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:37,969 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/465d1b7909fc4c858a12d7c5d9f31d29, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/54daea1dc15347dd9d3c9eae14fac69a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/254da17333c4431189f90fd6778e476f] 2024-11-18T03:16:37,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:37,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:37,970 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:37,970 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/e138c1389aee42838c0461ce69d7f8c2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/841d74d31d274fab9cf966d6f8518708, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/41e520b4524c4aa2af74e1fc5759f5e0] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=36.1 K 2024-11-18T03:16:37,970 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 465d1b7909fc4c858a12d7c5d9f31d29, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731899792837 2024-11-18T03:16:37,973 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54daea1dc15347dd9d3c9eae14fac69a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731899794098 2024-11-18T03:16:37,973 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e138c1389aee42838c0461ce69d7f8c2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731899792837 2024-11-18T03:16:37,977 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 841d74d31d274fab9cf966d6f8518708, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731899794098 2024-11-18T03:16:37,977 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254da17333c4431189f90fd6778e476f, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731899796713 2024-11-18T03:16:37,978 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 41e520b4524c4aa2af74e1fc5759f5e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731899796713 2024-11-18T03:16:37,988 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#372 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:37,993 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/ae5e42dabd4c4c5c8b6192330634d68e is 50, key is test_row_0/B:col10/1731899796839/Put/seqid=0 2024-11-18T03:16:37,998 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:38,025 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118a9de47717c6c48caa3e8930626830641_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:38,028 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118a9de47717c6c48caa3e8930626830641_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:38,028 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a9de47717c6c48caa3e8930626830641_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:38,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742269_1445 (size=12799) 2024-11-18T03:16:38,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742270_1446 (size=4469) 2024-11-18T03:16:38,081 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#373 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:38,081 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/339c8eaa46964cdda9d1f392599700ce is 175, key is test_row_0/A:col10/1731899796839/Put/seqid=0 2024-11-18T03:16:38,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742271_1447 (size=31753) 2024-11-18T03:16:38,120 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/339c8eaa46964cdda9d1f392599700ce as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/339c8eaa46964cdda9d1f392599700ce 2024-11-18T03:16:38,125 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into 339c8eaa46964cdda9d1f392599700ce(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:38,125 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:38,125 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=13, startTime=1731899797964; duration=0sec 2024-11-18T03:16:38,126 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:38,126 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:38,126 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:38,127 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:38,127 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:38,127 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:38,127 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a66fd744248d4b55ab8954b946878198, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b5b756f63c73495f861f1bd8ba448da5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8b151e51f77542c98a807baf70118b43] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=36.1 K 2024-11-18T03:16:38,127 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66fd744248d4b55ab8954b946878198, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731899792837 2024-11-18T03:16:38,128 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5b756f63c73495f861f1bd8ba448da5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731899794098 2024-11-18T03:16:38,128 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b151e51f77542c98a807baf70118b43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731899796713 2024-11-18T03:16:38,147 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:38,147 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/b8be88d3c43740caa0bc906d6843da84 is 50, key is test_row_0/C:col10/1731899796839/Put/seqid=0 2024-11-18T03:16:38,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742272_1448 (size=12799) 2024-11-18T03:16:38,177 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/b8be88d3c43740caa0bc906d6843da84 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b8be88d3c43740caa0bc906d6843da84 2024-11-18T03:16:38,183 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into b8be88d3c43740caa0bc906d6843da84(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:38,183 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:38,183 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=13, startTime=1731899797965; duration=0sec 2024-11-18T03:16:38,183 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:38,183 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:38,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-18T03:16:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:38,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118efe86a0ea96a41ea8dc3463ef8d62df6_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899797100/Put/seqid=0 2024-11-18T03:16:38,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899858331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899858333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899858335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899858347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899858349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742273_1449 (size=14994) 2024-11-18T03:16:38,364 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,373 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118efe86a0ea96a41ea8dc3463ef8d62df6_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118efe86a0ea96a41ea8dc3463ef8d62df6_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:38,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/d6325f0d10074c8f934b4dba0e01e83a, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:38,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/d6325f0d10074c8f934b4dba0e01e83a is 175, key is test_row_0/A:col10/1731899797100/Put/seqid=0 2024-11-18T03:16:38,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742274_1450 (size=39949) 2024-11-18T03:16:38,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/d6325f0d10074c8f934b4dba0e01e83a 2024-11-18T03:16:38,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f3586ddf8d30485bb2b32cd9520e84fc is 50, key is test_row_0/B:col10/1731899797100/Put/seqid=0 2024-11-18T03:16:38,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742275_1451 (size=12301) 2024-11-18T03:16:38,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f3586ddf8d30485bb2b32cd9520e84fc 2024-11-18T03:16:38,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/6a366a2e6d6d4538ad2d4c4aaada5595 is 50, key is test_row_0/C:col10/1731899797100/Put/seqid=0 2024-11-18T03:16:38,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742276_1452 (size=12301) 2024-11-18T03:16:38,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/6a366a2e6d6d4538ad2d4c4aaada5595 2024-11-18T03:16:38,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/d6325f0d10074c8f934b4dba0e01e83a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d6325f0d10074c8f934b4dba0e01e83a 2024-11-18T03:16:38,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d6325f0d10074c8f934b4dba0e01e83a, entries=200, sequenceid=291, filesize=39.0 K 2024-11-18T03:16:38,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/f3586ddf8d30485bb2b32cd9520e84fc as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f3586ddf8d30485bb2b32cd9520e84fc 2024-11-18T03:16:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f3586ddf8d30485bb2b32cd9520e84fc, entries=150, sequenceid=291, filesize=12.0 K 2024-11-18T03:16:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899858461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899858461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899858461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899858461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/6a366a2e6d6d4538ad2d4c4aaada5595 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6a366a2e6d6d4538ad2d4c4aaada5595 2024-11-18T03:16:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,477 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/ae5e42dabd4c4c5c8b6192330634d68e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/ae5e42dabd4c4c5c8b6192330634d68e 2024-11-18T03:16:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6a366a2e6d6d4538ad2d4c4aaada5595, entries=150, sequenceid=291, filesize=12.0 K 2024-11-18T03:16:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 70fa1092abe5ed71c7b0809f585c5469 in 169ms, sequenceid=291, compaction requested=false 2024-11-18T03:16:38,482 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into ae5e42dabd4c4c5c8b6192330634d68e(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:38,482 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:38,482 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=13, startTime=1731899797965; duration=0sec 2024-11-18T03:16:38,482 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:38,482 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:38,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:16:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b6c2a7c74c53415e92cdac1857d2441c_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899798716/Put/seqid=0 2024-11-18T03:16:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:38,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742277_1453 (size=12454) 2024-11-18T03:16:38,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899798780, max timestamp=1731899798804Num uniques 859; 2024-11-18T03:16:38,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:16:38,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899798780, max timestamp=1731899798804Num uniques 859; 2024-11-18T03:16:38,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:16:38,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731899798780, max timestamp=1731899798804Num uniques 859; 2024-11-18T03:16:38,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:16:38,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899858867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899858869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899858870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899858882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899858990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899858990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899859001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899859001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:39,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b6c2a7c74c53415e92cdac1857d2441c_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b6c2a7c74c53415e92cdac1857d2441c_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:39,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/ebf70dcaffb54afcb4a841350da55985, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:39,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/ebf70dcaffb54afcb4a841350da55985 is 175, key is test_row_0/A:col10/1731899798716/Put/seqid=0 2024-11-18T03:16:39,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742278_1454 (size=31255) 2024-11-18T03:16:39,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/ebf70dcaffb54afcb4a841350da55985 2024-11-18T03:16:39,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/0861080b200940c7a9ddca4c0ad4b27d is 50, key is test_row_0/B:col10/1731899798716/Put/seqid=0 2024-11-18T03:16:39,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742279_1455 (size=12301) 2024-11-18T03:16:39,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/0861080b200940c7a9ddca4c0ad4b27d 2024-11-18T03:16:39,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/844bb44104c942bbbe757571f4d87e10 is 50, key is test_row_0/C:col10/1731899798716/Put/seqid=0 2024-11-18T03:16:39,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742280_1456 (size=12301) 2024-11-18T03:16:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899859210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899859210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899859221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899859221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731899859361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,371 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 70fa1092abe5ed71c7b0809f585c5469 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899859521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899859521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899859540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899859540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:39,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/844bb44104c942bbbe757571f4d87e10 2024-11-18T03:16:39,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/ebf70dcaffb54afcb4a841350da55985 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ebf70dcaffb54afcb4a841350da55985 2024-11-18T03:16:39,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ebf70dcaffb54afcb4a841350da55985, entries=150, sequenceid=302, filesize=30.5 K 2024-11-18T03:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/0861080b200940c7a9ddca4c0ad4b27d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0861080b200940c7a9ddca4c0ad4b27d 2024-11-18T03:16:39,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0861080b200940c7a9ddca4c0ad4b27d, entries=150, sequenceid=302, filesize=12.0 K 2024-11-18T03:16:39,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/844bb44104c942bbbe757571f4d87e10 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/844bb44104c942bbbe757571f4d87e10 2024-11-18T03:16:39,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/844bb44104c942bbbe757571f4d87e10, entries=150, sequenceid=302, filesize=12.0 K 2024-11-18T03:16:39,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 70fa1092abe5ed71c7b0809f585c5469 in 892ms, sequenceid=302, compaction requested=true 2024-11-18T03:16:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:39,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:39,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:39,610 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:39,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:39,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:39,610 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:39,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:39,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:39,610 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:39,610 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:39,610 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:39,610 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:39,611 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:39,611 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:39,611 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/ae5e42dabd4c4c5c8b6192330634d68e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f3586ddf8d30485bb2b32cd9520e84fc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0861080b200940c7a9ddca4c0ad4b27d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=36.5 K 2024-11-18T03:16:39,611 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/339c8eaa46964cdda9d1f392599700ce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d6325f0d10074c8f934b4dba0e01e83a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ebf70dcaffb54afcb4a841350da55985] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=100.5 K 2024-11-18T03:16:39,611 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:39,611 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/339c8eaa46964cdda9d1f392599700ce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d6325f0d10074c8f934b4dba0e01e83a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ebf70dcaffb54afcb4a841350da55985] 2024-11-18T03:16:39,611 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ae5e42dabd4c4c5c8b6192330634d68e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731899796713 2024-11-18T03:16:39,611 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339c8eaa46964cdda9d1f392599700ce, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731899796713 2024-11-18T03:16:39,611 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f3586ddf8d30485bb2b32cd9520e84fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731899797069 2024-11-18T03:16:39,611 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0861080b200940c7a9ddca4c0ad4b27d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731899798345 2024-11-18T03:16:39,612 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6325f0d10074c8f934b4dba0e01e83a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731899797069 2024-11-18T03:16:39,615 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf70dcaffb54afcb4a841350da55985, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731899798345 2024-11-18T03:16:39,623 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:39,630 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:39,631 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/22f2788ad7c647dca347164fb2557511 is 50, key is test_row_0/B:col10/1731899798716/Put/seqid=0 2024-11-18T03:16:39,632 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411180f65dd3107864703851699e1b22445ad_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:39,634 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411180f65dd3107864703851699e1b22445ad_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:39,634 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180f65dd3107864703851699e1b22445ad_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:39,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742281_1457 (size=13051) 2024-11-18T03:16:39,648 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/22f2788ad7c647dca347164fb2557511 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/22f2788ad7c647dca347164fb2557511 2024-11-18T03:16:39,651 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into 22f2788ad7c647dca347164fb2557511(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:39,651 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:39,651 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=13, startTime=1731899799610; duration=0sec 2024-11-18T03:16:39,651 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:39,651 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:39,651 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:39,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742282_1458 (size=4469) 2024-11-18T03:16:39,655 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:39,655 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:39,655 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:39,656 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b8be88d3c43740caa0bc906d6843da84, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6a366a2e6d6d4538ad2d4c4aaada5595, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/844bb44104c942bbbe757571f4d87e10] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=36.5 K 2024-11-18T03:16:39,656 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#381 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:39,656 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b8be88d3c43740caa0bc906d6843da84, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731899796713 2024-11-18T03:16:39,656 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/59e33a6050aa4223977b6f181fbe313f is 175, key is test_row_0/A:col10/1731899798716/Put/seqid=0 2024-11-18T03:16:39,660 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a366a2e6d6d4538ad2d4c4aaada5595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731899797069 2024-11-18T03:16:39,660 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 844bb44104c942bbbe757571f4d87e10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731899798345 2024-11-18T03:16:39,670 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:39,670 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/226ab79c931d47d1be05cb34adea5352 is 50, key is test_row_0/C:col10/1731899798716/Put/seqid=0 2024-11-18T03:16:39,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742283_1459 (size=32005) 2024-11-18T03:16:39,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742284_1460 (size=13051) 2024-11-18T03:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:40,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T03:16:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:40,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118223c775040884d8089a4fd683c2b6598_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_0/A:col10/1731899798881/Put/seqid=0 2024-11-18T03:16:40,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742285_1461 (size=14994) 2024-11-18T03:16:40,068 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:40,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118223c775040884d8089a4fd683c2b6598_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118223c775040884d8089a4fd683c2b6598_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:40,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/42d37ef41f184d19a4536317acd846ec, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:40,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/42d37ef41f184d19a4536317acd846ec is 175, key is test_row_0/A:col10/1731899798881/Put/seqid=0 2024-11-18T03:16:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742286_1462 (size=39949) 2024-11-18T03:16:40,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899860069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899860070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899860070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899860077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,088 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/59e33a6050aa4223977b6f181fbe313f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/59e33a6050aa4223977b6f181fbe313f 2024-11-18T03:16:40,088 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/226ab79c931d47d1be05cb34adea5352 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/226ab79c931d47d1be05cb34adea5352 2024-11-18T03:16:40,092 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into 226ab79c931d47d1be05cb34adea5352(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:40,092 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:40,092 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=13, startTime=1731899799610; duration=0sec 2024-11-18T03:16:40,092 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:40,092 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:40,092 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into 59e33a6050aa4223977b6f181fbe313f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:40,092 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:40,092 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=13, startTime=1731899799610; duration=0sec 2024-11-18T03:16:40,092 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:40,092 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899860180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T03:16:40,191 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:40,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-18T03:16:40,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:40,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:40,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899860191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899860191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:40,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899860401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899860411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899860411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,479 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/42d37ef41f184d19a4536317acd846ec 2024-11-18T03:16:40,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/fcb6d80975a647cb90d709b919006bd8 is 50, key is test_row_0/B:col10/1731899798881/Put/seqid=0 2024-11-18T03:16:40,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742287_1463 (size=12301) 2024-11-18T03:16:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:40,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:40,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:40,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899860721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899860731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899860731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:40,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:40,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/fcb6d80975a647cb90d709b919006bd8 2024-11-18T03:16:40,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/521bae3ecceb443db4b3c2305a8c187b is 50, key is test_row_0/C:col10/1731899798881/Put/seqid=0 2024-11-18T03:16:40,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742288_1464 (size=12301) 2024-11-18T03:16:40,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:40,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:40,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43588 deadline: 1731899861091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:41,099 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in 70fa1092abe5ed71c7b0809f585c5469 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:16:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:41,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1731899861241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43568 deadline: 1731899861252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731899861253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:16:41,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:41,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:41,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/521bae3ecceb443db4b3c2305a8c187b 2024-11-18T03:16:41,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/42d37ef41f184d19a4536317acd846ec as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/42d37ef41f184d19a4536317acd846ec 2024-11-18T03:16:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:41,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/42d37ef41f184d19a4536317acd846ec, entries=200, sequenceid=340, filesize=39.0 K 2024-11-18T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/fcb6d80975a647cb90d709b919006bd8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/fcb6d80975a647cb90d709b919006bd8 2024-11-18T03:16:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/fcb6d80975a647cb90d709b919006bd8, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T03:16:41,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/521bae3ecceb443db4b3c2305a8c187b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/521bae3ecceb443db4b3c2305a8c187b 2024-11-18T03:16:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/521bae3ecceb443db4b3c2305a8c187b, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T03:16:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 70fa1092abe5ed71c7b0809f585c5469 in 1298ms, sequenceid=340, compaction requested=false 2024-11-18T03:16:41,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,378 INFO [Thread-1675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1675. 2024-11-18T03:16:41,378 DEBUG [Thread-1675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,378 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,378 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,379 INFO [Thread-1679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1679. 2024-11-18T03:16:41,379 DEBUG [Thread-1679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,379 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,379 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:16:41,385 INFO [Thread-1673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1673. 2024-11-18T03:16:41,385 DEBUG [Thread-1673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:41,385 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,385 INFO [Thread-1664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1664. 2024-11-18T03:16:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:41,385 DEBUG [Thread-1664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:41,385 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,385 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:41,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:41,385 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:41,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183993c0da30104ad0afcdce7a4ef7f491_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_1/A:col10/1731899801383/Put/seqid=0 2024-11-18T03:16:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742289_1465 (size=12454) 2024-11-18T03:16:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,410 INFO [Thread-1671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1671. 2024-11-18T03:16:41,410 DEBUG [Thread-1671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:41,410 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,410 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:41,411 INFO [Thread-1677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1677. 2024-11-18T03:16:41,411 DEBUG [Thread-1677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:41,411 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,411 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:41,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:41,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:41,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:41,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:41,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:41,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,800 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:41,803 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183993c0da30104ad0afcdce7a4ef7f491_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183993c0da30104ad0afcdce7a4ef7f491_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:41,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/52cc00303c75451a912f2c1e20009dfe, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:41,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/52cc00303c75451a912f2c1e20009dfe is 175, key is test_row_1/A:col10/1731899801383/Put/seqid=0 2024-11-18T03:16:41,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742290_1466 (size=31251) 2024-11-18T03:16:41,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:41,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:41,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:41,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,207 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/52cc00303c75451a912f2c1e20009dfe 2024-11-18T03:16:42,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c03d1f8804d24269b9b82d1fc608c580 is 50, key is test_row_1/B:col10/1731899801383/Put/seqid=0 2024-11-18T03:16:42,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742291_1467 (size=9857) 2024-11-18T03:16:42,217 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T03:16:42,262 INFO [Thread-1660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1660. 2024-11-18T03:16:42,263 DEBUG [Thread-1660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:42,263 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:42,263 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:42,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:42,273 INFO [Thread-1668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1668. 2024-11-18T03:16:42,273 DEBUG [Thread-1668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:42,273 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:42,273 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:42,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:42,284 INFO [Thread-1662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1662. 2024-11-18T03:16:42,284 DEBUG [Thread-1662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:42,284 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:42,284 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:42,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:42,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c03d1f8804d24269b9b82d1fc608c580 2024-11-18T03:16:42,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/f1ca2898cb1e431b80edcccebd57b917 is 50, key is test_row_1/C:col10/1731899801383/Put/seqid=0 2024-11-18T03:16:42,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742292_1468 (size=9857) 2024-11-18T03:16:42,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:42,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:42,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:42,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:42,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. as already flushing 2024-11-18T03:16:42,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:42,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:43,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/f1ca2898cb1e431b80edcccebd57b917 2024-11-18T03:16:43,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/52cc00303c75451a912f2c1e20009dfe as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/52cc00303c75451a912f2c1e20009dfe 2024-11-18T03:16:43,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/52cc00303c75451a912f2c1e20009dfe, entries=150, sequenceid=352, filesize=30.5 K 2024-11-18T03:16:43,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/c03d1f8804d24269b9b82d1fc608c580 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c03d1f8804d24269b9b82d1fc608c580 2024-11-18T03:16:43,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c03d1f8804d24269b9b82d1fc608c580, entries=100, sequenceid=352, filesize=9.6 K 2024-11-18T03:16:43,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/f1ca2898cb1e431b80edcccebd57b917 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/f1ca2898cb1e431b80edcccebd57b917 2024-11-18T03:16:43,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/f1ca2898cb1e431b80edcccebd57b917, entries=100, sequenceid=352, filesize=9.6 K 2024-11-18T03:16:43,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 70fa1092abe5ed71c7b0809f585c5469 in 1656ms, sequenceid=352, compaction requested=true 2024-11-18T03:16:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:43,042 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:43,042 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70fa1092abe5ed71c7b0809f585c5469:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/B is initiating minor compaction (all files) 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/A is initiating minor compaction (all files) 2024-11-18T03:16:43,043 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/B in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:43,043 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/A in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:43,043 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/59e33a6050aa4223977b6f181fbe313f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/42d37ef41f184d19a4536317acd846ec, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/52cc00303c75451a912f2c1e20009dfe] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=100.8 K 2024-11-18T03:16:43,043 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/22f2788ad7c647dca347164fb2557511, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/fcb6d80975a647cb90d709b919006bd8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c03d1f8804d24269b9b82d1fc608c580] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=34.4 K 2024-11-18T03:16:43,043 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/59e33a6050aa4223977b6f181fbe313f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/42d37ef41f184d19a4536317acd846ec, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/52cc00303c75451a912f2c1e20009dfe] 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 22f2788ad7c647dca347164fb2557511, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731899798345 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e33a6050aa4223977b6f181fbe313f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731899798345 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d37ef41f184d19a4536317acd846ec, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731899798847 2024-11-18T03:16:43,043 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb6d80975a647cb90d709b919006bd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731899798869 2024-11-18T03:16:43,044 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52cc00303c75451a912f2c1e20009dfe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731899800069 2024-11-18T03:16:43,044 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c03d1f8804d24269b9b82d1fc608c580, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731899800069 2024-11-18T03:16:43,049 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:43,050 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:43,050 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411182f453eef21e249b2badd57b83d787398_70fa1092abe5ed71c7b0809f585c5469 store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:43,051 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/08d3ec68bcf04bd9a205841692efa0b0 is 50, key is test_row_0/B:col10/1731899798881/Put/seqid=0 2024-11-18T03:16:43,054 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411182f453eef21e249b2badd57b83d787398_70fa1092abe5ed71c7b0809f585c5469, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:43,054 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182f453eef21e249b2badd57b83d787398_70fa1092abe5ed71c7b0809f585c5469 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:43,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742293_1469 (size=13153) 2024-11-18T03:16:43,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742294_1470 (size=4469) 2024-11-18T03:16:43,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T03:16:43,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:43,122 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 70fa1092abe5ed71c7b0809f585c5469 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T03:16:43,122 INFO [Thread-1666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1666. 2024-11-18T03:16:43,122 DEBUG [Thread-1666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:16:43,122 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:43,122 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:43,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:43,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=A 2024-11-18T03:16:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=B 2024-11-18T03:16:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70fa1092abe5ed71c7b0809f585c5469, store=C 2024-11-18T03:16:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ddc0de553aec4c368d0ee5eef77bdd45_70fa1092abe5ed71c7b0809f585c5469 is 50, key is test_row_1/A:col10/1731899802271/Put/seqid=0 2024-11-18T03:16:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742295_1471 (size=9914) 2024-11-18T03:16:43,463 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#A#compaction#391 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:43,464 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/1e9cc911cab846ae8c473aa7bdf11ba4 is 175, key is test_row_0/A:col10/1731899798881/Put/seqid=0 2024-11-18T03:16:43,476 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/08d3ec68bcf04bd9a205841692efa0b0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/08d3ec68bcf04bd9a205841692efa0b0 2024-11-18T03:16:43,483 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/B of 70fa1092abe5ed71c7b0809f585c5469 into 08d3ec68bcf04bd9a205841692efa0b0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:43,483 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:43,483 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/B, priority=13, startTime=1731899803042; duration=0sec 2024-11-18T03:16:43,483 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:43,483 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:B 2024-11-18T03:16:43,483 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:43,484 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:43,484 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 70fa1092abe5ed71c7b0809f585c5469/C is initiating minor compaction (all files) 2024-11-18T03:16:43,484 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70fa1092abe5ed71c7b0809f585c5469/C in TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:43,485 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/226ab79c931d47d1be05cb34adea5352, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/521bae3ecceb443db4b3c2305a8c187b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/f1ca2898cb1e431b80edcccebd57b917] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp, totalSize=34.4 K 2024-11-18T03:16:43,485 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 226ab79c931d47d1be05cb34adea5352, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731899798345 2024-11-18T03:16:43,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742296_1472 (size=32214) 2024-11-18T03:16:43,485 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 521bae3ecceb443db4b3c2305a8c187b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731899798869 2024-11-18T03:16:43,485 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ca2898cb1e431b80edcccebd57b917, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731899800069 2024-11-18T03:16:43,491 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70fa1092abe5ed71c7b0809f585c5469#C#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:43,492 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/46c3c5883b5b4932993f2ad61f7d377d is 50, key is test_row_0/C:col10/1731899798881/Put/seqid=0 2024-11-18T03:16:43,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742297_1473 (size=13153) 2024-11-18T03:16:43,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:43,536 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ddc0de553aec4c368d0ee5eef77bdd45_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ddc0de553aec4c368d0ee5eef77bdd45_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:43,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06efbc07dcbd4cfaba9ce68def930b60, store: [table=TestAcidGuarantees family=A region=70fa1092abe5ed71c7b0809f585c5469] 2024-11-18T03:16:43,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06efbc07dcbd4cfaba9ce68def930b60 is 175, key is test_row_1/A:col10/1731899802271/Put/seqid=0 2024-11-18T03:16:43,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742298_1474 (size=22561) 2024-11-18T03:16:43,894 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/1e9cc911cab846ae8c473aa7bdf11ba4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1e9cc911cab846ae8c473aa7bdf11ba4 2024-11-18T03:16:43,898 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/A of 70fa1092abe5ed71c7b0809f585c5469 into 1e9cc911cab846ae8c473aa7bdf11ba4(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:43,898 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:43,898 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/A, priority=13, startTime=1731899803042; duration=0sec 2024-11-18T03:16:43,898 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:43,899 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:A 2024-11-18T03:16:43,903 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/46c3c5883b5b4932993f2ad61f7d377d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/46c3c5883b5b4932993f2ad61f7d377d 2024-11-18T03:16:43,907 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70fa1092abe5ed71c7b0809f585c5469/C of 70fa1092abe5ed71c7b0809f585c5469 into 46c3c5883b5b4932993f2ad61f7d377d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:43,907 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:43,907 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469., storeName=70fa1092abe5ed71c7b0809f585c5469/C, priority=13, startTime=1731899803042; duration=0sec 2024-11-18T03:16:43,907 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:43,907 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70fa1092abe5ed71c7b0809f585c5469:C 2024-11-18T03:16:43,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06efbc07dcbd4cfaba9ce68def930b60 2024-11-18T03:16:43,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/3c9487c994fb48b3913f1bf822e21544 is 50, key is test_row_1/B:col10/1731899802271/Put/seqid=0 2024-11-18T03:16:43,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742299_1475 (size=9857) 2024-11-18T03:16:44,354 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/3c9487c994fb48b3913f1bf822e21544 2024-11-18T03:16:44,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3782df45aa6d46d6a70da68ce73562c1 is 50, key is test_row_1/C:col10/1731899802271/Put/seqid=0 2024-11-18T03:16:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:44,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742300_1476 (size=9857) 2024-11-18T03:16:44,773 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3782df45aa6d46d6a70da68ce73562c1 2024-11-18T03:16:44,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/A/06efbc07dcbd4cfaba9ce68def930b60 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06efbc07dcbd4cfaba9ce68def930b60 2024-11-18T03:16:44,780 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06efbc07dcbd4cfaba9ce68def930b60, entries=100, sequenceid=359, filesize=22.0 K 2024-11-18T03:16:44,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/B/3c9487c994fb48b3913f1bf822e21544 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/3c9487c994fb48b3913f1bf822e21544 2024-11-18T03:16:44,783 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/3c9487c994fb48b3913f1bf822e21544, entries=100, sequenceid=359, filesize=9.6 K 2024-11-18T03:16:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/.tmp/C/3782df45aa6d46d6a70da68ce73562c1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3782df45aa6d46d6a70da68ce73562c1 2024-11-18T03:16:44,788 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3782df45aa6d46d6a70da68ce73562c1, entries=100, sequenceid=359, filesize=9.6 K 2024-11-18T03:16:44,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 70fa1092abe5ed71c7b0809f585c5469 in 1667ms, sequenceid=359, compaction requested=false 2024-11-18T03:16:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 70fa1092abe5ed71c7b0809f585c5469: 2024-11-18T03:16:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-18T03:16:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-18T03:16:44,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-18T03:16:44,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5960 sec 2024-11-18T03:16:44,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 4.6010 sec 2024-11-18T03:16:47,513 WARN [regionserver/fd862c3a04f9:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-18T03:16:47,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/b44ccb4cb3684ed19f2ff39e573b039d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/19c6ea17b9b943158d6ec48cc4a164a0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/0586d60dc9224a86930d68810eea8bcb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d81729eb77794c6ba8d0046147bcbef3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/29393b11b63e4fbfaedbabd788758719, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1aba114bbf4848689523f69f8cbb3b60, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/8ae818cfa3f54da9b2c4bd2702c72bfa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/993ffe2619914ad7bbe6a6f77c7fb65c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06bb5b847fa2425dae52392cd4ebf096, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/9b9677c13b5d4a82a3c283820ed7ea3e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/31836cb9aac242548c6236ec84687f2e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/08cd1765a51d4c03bc4f77e65507e1d8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/7b06ba4e1cfd488ab2e5dd7b221d17f1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ce794658b0b049d29dfc85ef611dd1e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/5ffa5b6ae308484397c2e0fcec68468b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/3aac6a02ae734420afc9c47b0153a4dd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/465d1b7909fc4c858a12d7c5d9f31d29, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/54daea1dc15347dd9d3c9eae14fac69a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/254da17333c4431189f90fd6778e476f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/339c8eaa46964cdda9d1f392599700ce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d6325f0d10074c8f934b4dba0e01e83a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/59e33a6050aa4223977b6f181fbe313f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ebf70dcaffb54afcb4a841350da55985, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/42d37ef41f184d19a4536317acd846ec, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/52cc00303c75451a912f2c1e20009dfe] to archive 2024-11-18T03:16:47,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:16:47,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/b44ccb4cb3684ed19f2ff39e573b039d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/b44ccb4cb3684ed19f2ff39e573b039d 2024-11-18T03:16:47,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/19c6ea17b9b943158d6ec48cc4a164a0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/19c6ea17b9b943158d6ec48cc4a164a0 2024-11-18T03:16:47,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/0586d60dc9224a86930d68810eea8bcb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/0586d60dc9224a86930d68810eea8bcb 2024-11-18T03:16:47,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d81729eb77794c6ba8d0046147bcbef3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d81729eb77794c6ba8d0046147bcbef3 2024-11-18T03:16:47,520 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/29393b11b63e4fbfaedbabd788758719 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/29393b11b63e4fbfaedbabd788758719 2024-11-18T03:16:47,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1aba114bbf4848689523f69f8cbb3b60 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1aba114bbf4848689523f69f8cbb3b60 2024-11-18T03:16:47,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/8ae818cfa3f54da9b2c4bd2702c72bfa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/8ae818cfa3f54da9b2c4bd2702c72bfa 2024-11-18T03:16:47,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/993ffe2619914ad7bbe6a6f77c7fb65c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/993ffe2619914ad7bbe6a6f77c7fb65c 2024-11-18T03:16:47,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06bb5b847fa2425dae52392cd4ebf096 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06bb5b847fa2425dae52392cd4ebf096 2024-11-18T03:16:47,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/9b9677c13b5d4a82a3c283820ed7ea3e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/9b9677c13b5d4a82a3c283820ed7ea3e 2024-11-18T03:16:47,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/31836cb9aac242548c6236ec84687f2e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/31836cb9aac242548c6236ec84687f2e 2024-11-18T03:16:47,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/08cd1765a51d4c03bc4f77e65507e1d8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/08cd1765a51d4c03bc4f77e65507e1d8 2024-11-18T03:16:47,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/7b06ba4e1cfd488ab2e5dd7b221d17f1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/7b06ba4e1cfd488ab2e5dd7b221d17f1 2024-11-18T03:16:47,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ce794658b0b049d29dfc85ef611dd1e2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ce794658b0b049d29dfc85ef611dd1e2 2024-11-18T03:16:47,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/5ffa5b6ae308484397c2e0fcec68468b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/5ffa5b6ae308484397c2e0fcec68468b 2024-11-18T03:16:47,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/3aac6a02ae734420afc9c47b0153a4dd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/3aac6a02ae734420afc9c47b0153a4dd 2024-11-18T03:16:47,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/465d1b7909fc4c858a12d7c5d9f31d29 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/465d1b7909fc4c858a12d7c5d9f31d29 2024-11-18T03:16:47,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/54daea1dc15347dd9d3c9eae14fac69a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/54daea1dc15347dd9d3c9eae14fac69a 2024-11-18T03:16:47,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/254da17333c4431189f90fd6778e476f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/254da17333c4431189f90fd6778e476f 2024-11-18T03:16:47,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/339c8eaa46964cdda9d1f392599700ce to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/339c8eaa46964cdda9d1f392599700ce 2024-11-18T03:16:47,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d6325f0d10074c8f934b4dba0e01e83a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/d6325f0d10074c8f934b4dba0e01e83a 2024-11-18T03:16:47,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/59e33a6050aa4223977b6f181fbe313f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/59e33a6050aa4223977b6f181fbe313f 2024-11-18T03:16:47,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ebf70dcaffb54afcb4a841350da55985 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/ebf70dcaffb54afcb4a841350da55985 2024-11-18T03:16:47,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/42d37ef41f184d19a4536317acd846ec to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/42d37ef41f184d19a4536317acd846ec 2024-11-18T03:16:47,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/52cc00303c75451a912f2c1e20009dfe to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/52cc00303c75451a912f2c1e20009dfe 2024-11-18T03:16:47,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d2bddd611f30487d8d7ca6bfaac43951, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/39d4fbba02ae4621927fefbd1d783f51, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/66f5a4aac69448359cd0250fdd25c880, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/16b467701c574cb3a01b7977412ef1bd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/4194d27677ee46c29649ebe552fe8282, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c7f83820fb034d3c92dcc21766c65589, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0ba64ec17b524219a02074cf11816974, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/23920e8d469a450aa0575ea4720df547, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f155f2b7908944e49771203846ed090f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/b660c397e8614301aca6dcb2b9a2e12a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c685a4eee2594848a17fc70dc44b61e4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d4b09d3032a8415a91f11a8cf10a497a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f7c7dcc95ae6450fbde86332ab6d868b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/06b0995226134166afa684d3f48cad1b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/42e074495a0547c5a14e50572920b9e9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/e138c1389aee42838c0461ce69d7f8c2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/704eeafc34344989a60e758f5673540a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/841d74d31d274fab9cf966d6f8518708, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/ae5e42dabd4c4c5c8b6192330634d68e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/41e520b4524c4aa2af74e1fc5759f5e0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f3586ddf8d30485bb2b32cd9520e84fc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/22f2788ad7c647dca347164fb2557511, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0861080b200940c7a9ddca4c0ad4b27d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/fcb6d80975a647cb90d709b919006bd8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c03d1f8804d24269b9b82d1fc608c580] to archive 2024-11-18T03:16:47,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:16:47,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d2bddd611f30487d8d7ca6bfaac43951 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d2bddd611f30487d8d7ca6bfaac43951 2024-11-18T03:16:47,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/39d4fbba02ae4621927fefbd1d783f51 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/39d4fbba02ae4621927fefbd1d783f51 2024-11-18T03:16:47,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/66f5a4aac69448359cd0250fdd25c880 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/66f5a4aac69448359cd0250fdd25c880 2024-11-18T03:16:47,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/16b467701c574cb3a01b7977412ef1bd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/16b467701c574cb3a01b7977412ef1bd 2024-11-18T03:16:47,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/4194d27677ee46c29649ebe552fe8282 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/4194d27677ee46c29649ebe552fe8282 2024-11-18T03:16:47,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c7f83820fb034d3c92dcc21766c65589 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c7f83820fb034d3c92dcc21766c65589 2024-11-18T03:16:47,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0ba64ec17b524219a02074cf11816974 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0ba64ec17b524219a02074cf11816974 2024-11-18T03:16:47,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/23920e8d469a450aa0575ea4720df547 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/23920e8d469a450aa0575ea4720df547 2024-11-18T03:16:47,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f155f2b7908944e49771203846ed090f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f155f2b7908944e49771203846ed090f 2024-11-18T03:16:47,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/b660c397e8614301aca6dcb2b9a2e12a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/b660c397e8614301aca6dcb2b9a2e12a 2024-11-18T03:16:47,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c685a4eee2594848a17fc70dc44b61e4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c685a4eee2594848a17fc70dc44b61e4 2024-11-18T03:16:47,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d4b09d3032a8415a91f11a8cf10a497a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/d4b09d3032a8415a91f11a8cf10a497a 2024-11-18T03:16:47,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f7c7dcc95ae6450fbde86332ab6d868b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f7c7dcc95ae6450fbde86332ab6d868b 2024-11-18T03:16:47,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/06b0995226134166afa684d3f48cad1b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/06b0995226134166afa684d3f48cad1b 2024-11-18T03:16:47,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/42e074495a0547c5a14e50572920b9e9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/42e074495a0547c5a14e50572920b9e9 2024-11-18T03:16:47,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/e138c1389aee42838c0461ce69d7f8c2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/e138c1389aee42838c0461ce69d7f8c2 2024-11-18T03:16:47,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/704eeafc34344989a60e758f5673540a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/704eeafc34344989a60e758f5673540a 2024-11-18T03:16:47,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/841d74d31d274fab9cf966d6f8518708 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/841d74d31d274fab9cf966d6f8518708 2024-11-18T03:16:47,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/ae5e42dabd4c4c5c8b6192330634d68e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/ae5e42dabd4c4c5c8b6192330634d68e 2024-11-18T03:16:47,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/41e520b4524c4aa2af74e1fc5759f5e0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/41e520b4524c4aa2af74e1fc5759f5e0 2024-11-18T03:16:47,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f3586ddf8d30485bb2b32cd9520e84fc to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/f3586ddf8d30485bb2b32cd9520e84fc 2024-11-18T03:16:47,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/22f2788ad7c647dca347164fb2557511 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/22f2788ad7c647dca347164fb2557511 2024-11-18T03:16:47,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0861080b200940c7a9ddca4c0ad4b27d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/0861080b200940c7a9ddca4c0ad4b27d 2024-11-18T03:16:47,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/fcb6d80975a647cb90d709b919006bd8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/fcb6d80975a647cb90d709b919006bd8 2024-11-18T03:16:47,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c03d1f8804d24269b9b82d1fc608c580 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/c03d1f8804d24269b9b82d1fc608c580 2024-11-18T03:16:47,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d86f852969f4fc79eedc756f424b532, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8348b4a1c0a1426a9a159913690c9451, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9fbaf57efbf1438ca3e2580a810c5b78, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/4f3ec6359f9f472a9d331c513ad97b40, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/d53113d8513943d0bbb2b4358476e0d6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c60247e5e2d14131b9f8c96c0982bea6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74a62601c20c44098283113122873ea9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9eb2febcb742408a9d8f6290c2d7da31, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/e269469acea24a6f8d017adc50baebf9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6f1ced17295b4181baf67397f81527d5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c2a5a5f7e09847769c19dd85644eb948, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/0ff6f04d971c4e1cbfadc995b5045974, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a90c1787c9804b38a47341792e8f9d49, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74f7885647c940cd9a6943627d552f1d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d5520e71de74b3687568d501d20fb95, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a66fd744248d4b55ab8954b946878198, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/7b1ba919316b4b428eab60adeb6ea2d3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b5b756f63c73495f861f1bd8ba448da5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b8be88d3c43740caa0bc906d6843da84, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8b151e51f77542c98a807baf70118b43, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6a366a2e6d6d4538ad2d4c4aaada5595, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/226ab79c931d47d1be05cb34adea5352, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/844bb44104c942bbbe757571f4d87e10, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/521bae3ecceb443db4b3c2305a8c187b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/f1ca2898cb1e431b80edcccebd57b917] to archive 2024-11-18T03:16:47,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:16:47,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d86f852969f4fc79eedc756f424b532 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d86f852969f4fc79eedc756f424b532 2024-11-18T03:16:47,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8348b4a1c0a1426a9a159913690c9451 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8348b4a1c0a1426a9a159913690c9451 2024-11-18T03:16:47,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9fbaf57efbf1438ca3e2580a810c5b78 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9fbaf57efbf1438ca3e2580a810c5b78 2024-11-18T03:16:47,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/4f3ec6359f9f472a9d331c513ad97b40 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/4f3ec6359f9f472a9d331c513ad97b40 2024-11-18T03:16:47,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/d53113d8513943d0bbb2b4358476e0d6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/d53113d8513943d0bbb2b4358476e0d6 2024-11-18T03:16:47,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c60247e5e2d14131b9f8c96c0982bea6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c60247e5e2d14131b9f8c96c0982bea6 2024-11-18T03:16:47,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74a62601c20c44098283113122873ea9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74a62601c20c44098283113122873ea9 2024-11-18T03:16:47,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9eb2febcb742408a9d8f6290c2d7da31 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/9eb2febcb742408a9d8f6290c2d7da31 2024-11-18T03:16:47,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/e269469acea24a6f8d017adc50baebf9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/e269469acea24a6f8d017adc50baebf9 2024-11-18T03:16:47,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6f1ced17295b4181baf67397f81527d5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6f1ced17295b4181baf67397f81527d5 2024-11-18T03:16:47,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c2a5a5f7e09847769c19dd85644eb948 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/c2a5a5f7e09847769c19dd85644eb948 2024-11-18T03:16:47,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/0ff6f04d971c4e1cbfadc995b5045974 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/0ff6f04d971c4e1cbfadc995b5045974 2024-11-18T03:16:47,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a90c1787c9804b38a47341792e8f9d49 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a90c1787c9804b38a47341792e8f9d49 2024-11-18T03:16:47,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74f7885647c940cd9a6943627d552f1d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/74f7885647c940cd9a6943627d552f1d 2024-11-18T03:16:47,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d5520e71de74b3687568d501d20fb95 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3d5520e71de74b3687568d501d20fb95 2024-11-18T03:16:47,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a66fd744248d4b55ab8954b946878198 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/a66fd744248d4b55ab8954b946878198 2024-11-18T03:16:47,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/7b1ba919316b4b428eab60adeb6ea2d3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/7b1ba919316b4b428eab60adeb6ea2d3 2024-11-18T03:16:47,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b5b756f63c73495f861f1bd8ba448da5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b5b756f63c73495f861f1bd8ba448da5 2024-11-18T03:16:47,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b8be88d3c43740caa0bc906d6843da84 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/b8be88d3c43740caa0bc906d6843da84 2024-11-18T03:16:47,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8b151e51f77542c98a807baf70118b43 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/8b151e51f77542c98a807baf70118b43 2024-11-18T03:16:47,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6a366a2e6d6d4538ad2d4c4aaada5595 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/6a366a2e6d6d4538ad2d4c4aaada5595 2024-11-18T03:16:47,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/226ab79c931d47d1be05cb34adea5352 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/226ab79c931d47d1be05cb34adea5352 2024-11-18T03:16:47,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/844bb44104c942bbbe757571f4d87e10 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/844bb44104c942bbbe757571f4d87e10 2024-11-18T03:16:47,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/521bae3ecceb443db4b3c2305a8c187b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/521bae3ecceb443db4b3c2305a8c187b 2024-11-18T03:16:47,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fd862c3a04f9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/f1ca2898cb1e431b80edcccebd57b917 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/f1ca2898cb1e431b80edcccebd57b917 2024-11-18T03:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T03:16:48,382 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 562 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1686 rows 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 566 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1695 rows 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 563 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1680 rows 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 570 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1707 rows 2024-11-18T03:16:48,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 565 2024-11-18T03:16:48,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1692 rows 2024-11-18T03:16:48,383 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:16:48,383 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:16:48,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:48,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:48,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:16:48,385 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-18T03:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T03:16:48,389 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899808388"}]},"ts":"1731899808388"} 2024-11-18T03:16:48,390 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T03:16:48,390 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T03:16:48,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:16:48,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, UNASSIGN}] 2024-11-18T03:16:48,393 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, UNASSIGN 2024-11-18T03:16:48,394 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:48,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, UNASSIGN because future has completed 2024-11-18T03:16:48,395 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:16:48,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T03:16:48,547 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,547 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:16:48,547 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 70fa1092abe5ed71c7b0809f585c5469, disabling compactions & flushes 2024-11-18T03:16:48,547 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:48,547 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:48,547 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. after waiting 0 ms 2024-11-18T03:16:48,547 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:48,554 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/recovered.edits/365.seqid, newMaxSeqId=365, maxSeqId=4 2024-11-18T03:16:48,555 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469. 2024-11-18T03:16:48,555 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 70fa1092abe5ed71c7b0809f585c5469: Waiting for close lock at 1731899808547Running coprocessor pre-close hooks at 1731899808547Disabling compacts and flushes for region at 1731899808547Disabling writes for close at 1731899808547Writing region close event to WAL at 1731899808551 (+4 ms)Running coprocessor post-close hooks at 1731899808555 (+4 ms)Closed at 1731899808555 2024-11-18T03:16:48,557 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,557 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=70fa1092abe5ed71c7b0809f585c5469, regionState=CLOSED 2024-11-18T03:16:48,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:16:48,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-18T03:16:48,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 70fa1092abe5ed71c7b0809f585c5469, server=fd862c3a04f9,38549,1731899684479 in 165 msec 2024-11-18T03:16:48,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-18T03:16:48,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70fa1092abe5ed71c7b0809f585c5469, UNASSIGN in 169 msec 2024-11-18T03:16:48,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-18T03:16:48,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 173 msec 2024-11-18T03:16:48,567 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899808566"}]},"ts":"1731899808566"} 2024-11-18T03:16:48,568 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T03:16:48,568 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T03:16:48,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 184 msec 2024-11-18T03:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T03:16:48,701 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:48,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-18T03:16:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,703 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T03:16:48,703 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,705 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,707 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/recovered.edits] 2024-11-18T03:16:48,709 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06efbc07dcbd4cfaba9ce68def930b60 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/06efbc07dcbd4cfaba9ce68def930b60 2024-11-18T03:16:48,710 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1e9cc911cab846ae8c473aa7bdf11ba4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/A/1e9cc911cab846ae8c473aa7bdf11ba4 2024-11-18T03:16:48,712 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/08d3ec68bcf04bd9a205841692efa0b0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/08d3ec68bcf04bd9a205841692efa0b0 2024-11-18T03:16:48,713 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/3c9487c994fb48b3913f1bf822e21544 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/B/3c9487c994fb48b3913f1bf822e21544 2024-11-18T03:16:48,715 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3782df45aa6d46d6a70da68ce73562c1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/3782df45aa6d46d6a70da68ce73562c1 2024-11-18T03:16:48,716 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/46c3c5883b5b4932993f2ad61f7d377d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/C/46c3c5883b5b4932993f2ad61f7d377d 2024-11-18T03:16:48,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/recovered.edits/365.seqid to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469/recovered.edits/365.seqid 2024-11-18T03:16:48,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,718 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T03:16:48,719 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T03:16:48,720 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T03:16:48,722 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118088a7dc83ead444085aa57395e3a90f9_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118088a7dc83ead444085aa57395e3a90f9_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,724 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118223c775040884d8089a4fd683c2b6598_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118223c775040884d8089a4fd683c2b6598_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,724 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182359a29bf9d8456ba6118e01e4184fb2_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182359a29bf9d8456ba6118e01e4184fb2_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,725 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828669a8f00ad4c0da88c590be95d1223_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828669a8f00ad4c0da88c590be95d1223_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,726 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183993c0da30104ad0afcdce7a4ef7f491_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183993c0da30104ad0afcdce7a4ef7f491_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,727 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e5161d97f3c46958f08df2a1716b298_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e5161d97f3c46958f08df2a1716b298_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,728 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111853d787bda1ac49a7b9f8fa81ed29b64b_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111853d787bda1ac49a7b9f8fa81ed29b64b_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,728 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187dfa4f04d4734816bbc999e7d32b4c51_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187dfa4f04d4734816bbc999e7d32b4c51_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,729 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118937759576f2f4b95b098369f67655bdb_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118937759576f2f4b95b098369f67655bdb_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,730 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189996f571d8f246989a8176c9e6bb4274_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189996f571d8f246989a8176c9e6bb4274_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,730 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b6c2a7c74c53415e92cdac1857d2441c_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b6c2a7c74c53415e92cdac1857d2441c_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,731 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b939a9d0e0304c11b03c9d885d150bd9_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b939a9d0e0304c11b03c9d885d150bd9_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,732 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bcc0d9b81a754e7ca65bdd4a49db4fbe_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bcc0d9b81a754e7ca65bdd4a49db4fbe_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,733 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ddc0de553aec4c368d0ee5eef77bdd45_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ddc0de553aec4c368d0ee5eef77bdd45_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,733 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e280d63c74a94bdca581691fe56ee712_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e280d63c74a94bdca581691fe56ee712_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,734 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118efe86a0ea96a41ea8dc3463ef8d62df6_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118efe86a0ea96a41ea8dc3463ef8d62df6_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,735 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f7fb6151a51241f09bf6357b8c213498_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f7fb6151a51241f09bf6357b8c213498_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,736 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9831c03685c42ff9bbb991ae653056c_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9831c03685c42ff9bbb991ae653056c_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,736 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9df5585046942a6aa1561aa2a017ec1_70fa1092abe5ed71c7b0809f585c5469 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9df5585046942a6aa1561aa2a017ec1_70fa1092abe5ed71c7b0809f585c5469 2024-11-18T03:16:48,737 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T03:16:48,738 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,740 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T03:16:48,742 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T03:16:48,742 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,743 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T03:16:48,743 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731899808743"}]},"ts":"9223372036854775807"} 2024-11-18T03:16:48,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T03:16:48,744 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 70fa1092abe5ed71c7b0809f585c5469, NAME => 'TestAcidGuarantees,,1731899779209.70fa1092abe5ed71c7b0809f585c5469.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T03:16:48,744 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T03:16:48,744 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731899808744"}]},"ts":"9223372036854775807"} 2024-11-18T03:16:48,746 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T03:16:48,746 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-18T03:16:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T03:16:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T03:16:48,812 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:48,821 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 248), OpenFileDescriptor=460 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=997 (was 816) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2307 (was 3433) 2024-11-18T03:16:48,828 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=460, MaxFileDescriptor=1048576, SystemLoadAverage=997, ProcessCount=11, AvailableMemoryMB=2306 2024-11-18T03:16:48,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:16:48,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:16:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T03:16:48,832 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:16:48,832 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:48,832 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-18T03:16:48,832 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:16:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-18T03:16:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742301_1477 (size=1061) 2024-11-18T03:16:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-18T03:16:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-18T03:16:49,239 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:16:49,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742302_1478 (size=53) 2024-11-18T03:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-18T03:16:49,644 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:16:49,644 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9d402698952bfb73679bd006311c3148, disabling compactions & flushes 2024-11-18T03:16:49,644 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,644 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,644 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. after waiting 0 ms 2024-11-18T03:16:49,644 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,644 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,644 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d402698952bfb73679bd006311c3148: Waiting for close lock at 1731899809644Disabling compacts and flushes for region at 1731899809644Disabling writes for close at 1731899809644Writing region close event to WAL at 1731899809644Closed at 1731899809644 2024-11-18T03:16:49,645 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:16:49,645 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731899809645"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731899809645"}]},"ts":"1731899809645"} 2024-11-18T03:16:49,646 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:16:49,647 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:16:49,647 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899809647"}]},"ts":"1731899809647"} 2024-11-18T03:16:49,648 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T03:16:49,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, ASSIGN}] 2024-11-18T03:16:49,649 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, ASSIGN 2024-11-18T03:16:49,649 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, ASSIGN; state=OFFLINE, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=false 2024-11-18T03:16:49,801 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9d402698952bfb73679bd006311c3148, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:49,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, ASSIGN because future has completed 2024-11-18T03:16:49,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:16:49,962 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,962 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:16:49,963 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,963 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:16:49,963 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,963 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,964 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,965 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:49,965 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d402698952bfb73679bd006311c3148 columnFamilyName A 2024-11-18T03:16:49,965 DEBUG [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:49,966 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.HStore(327): Store=9d402698952bfb73679bd006311c3148/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:49,966 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,967 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:49,967 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d402698952bfb73679bd006311c3148 columnFamilyName B 2024-11-18T03:16:49,967 DEBUG [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:49,967 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.HStore(327): Store=9d402698952bfb73679bd006311c3148/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:49,967 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,968 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:16:49,968 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d402698952bfb73679bd006311c3148 columnFamilyName C 2024-11-18T03:16:49,968 DEBUG [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:16:49,969 INFO [StoreOpener-9d402698952bfb73679bd006311c3148-1 {}] regionserver.HStore(327): Store=9d402698952bfb73679bd006311c3148/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:16:49,969 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,969 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,969 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,970 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,970 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,970 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,971 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:16:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-18T03:16:49,972 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,974 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:16:49,974 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 9d402698952bfb73679bd006311c3148; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64307001, jitterRate=-0.041751012206077576}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:16:49,974 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:49,975 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 9d402698952bfb73679bd006311c3148: Running coprocessor pre-open hook at 1731899809963Writing region info on filesystem at 1731899809963Initializing all the Stores at 1731899809964 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899809964Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899809964Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899809964Cleaning up temporary data from old regions at 1731899809970 (+6 ms)Running coprocessor post-open hooks at 1731899809974 (+4 ms)Region opened successfully at 1731899809975 (+1 ms) 2024-11-18T03:16:49,976 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., pid=114, masterSystemTime=1731899809959 2024-11-18T03:16:49,977 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,978 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:49,978 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9d402698952bfb73679bd006311c3148, regionState=OPEN, openSeqNum=2, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:49,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:16:49,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-18T03:16:49,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 in 175 msec 2024-11-18T03:16:49,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-18T03:16:49,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, ASSIGN in 334 msec 2024-11-18T03:16:49,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:16:49,984 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899809984"}]},"ts":"1731899809984"} 2024-11-18T03:16:49,985 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T03:16:49,986 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:16:49,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-11-18T03:16:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-18T03:16:50,982 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:50,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b386e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:50,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:50,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:50,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5115c705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:50,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c815d63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:50,988 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,006 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:16:51,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fd862c3a04f9,45051,1731899683325 2024-11-18T03:16:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25cbad0 2024-11-18T03:16:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:16:51,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:16:51,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621cfad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,016 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23840c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,021 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2835f89f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27842489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f62fe14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@878a723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3464d23e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,037 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703fbb65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d78429, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe65078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,051 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e68e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,052 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,057 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f7d744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd9b86d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,067 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e0f987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,073 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1898a3c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48bc9769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,096 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f690ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,102 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135f9cd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460a8c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,112 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34268cb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,121 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6810a2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b76d744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edbcce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,140 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb39424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694dc39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,146 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a0b96b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,152 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687042ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f57cd4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:16:51,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:16:51,165 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:16:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:16:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:16:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2347b8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:16:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:16:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:16:51,170 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:16:51,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56846139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:16:51,171 DEBUG [Thread-2146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,172 DEBUG [Thread-2142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,175 DEBUG [Thread-2133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,178 DEBUG [Thread-2133 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,178 DEBUG [Thread-2133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,181 DEBUG [Thread-2135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,184 DEBUG [Thread-2148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,192 DEBUG [Thread-2137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,194 DEBUG [Thread-2137 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,199 DEBUG [Thread-2137 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,201 DEBUG [Thread-2150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,202 DEBUG [Thread-2139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,204 DEBUG [Thread-2131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,206 DEBUG [Thread-2150 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,206 DEBUG [Thread-2150 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,208 DEBUG [Thread-2131 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,209 DEBUG [Thread-2131 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-18T03:16:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T03:16:51,222 DEBUG [Thread-2144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:16:51,224 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:51,226 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:51,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:51,228 DEBUG [Thread-2150 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:16:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:16:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:16:51,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:16:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2] 2024-11-18T03:16:51,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:51,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:51,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:51,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:51,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:51,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:51,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/d15272a30aeb427fb9a3a67306c2d622 is 50, key is test_row_0/A:col10/1731899811214/Put/seqid=0 2024-11-18T03:16:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T03:16:51,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742303_1479 (size=12001) 2024-11-18T03:16:51,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/d15272a30aeb427fb9a3a67306c2d622 2024-11-18T03:16:51,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899871343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899871346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899811314, max timestamp=1731899811372Num uniques 859; 2024-11-18T03:16:51,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899811314, max timestamp=1731899811372Num uniques 859; 2024-11-18T03:16:51,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:51,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899811314, max timestamp=1731899811372Num uniques 859; 2024-11-18T03:16:51,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:16:51,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T03:16:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7480a22614804c0b83edea2caf80a752 is 50, key is test_row_0/B:col10/1731899811214/Put/seqid=0 2024-11-18T03:16:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742304_1480 (size=12001) 2024-11-18T03:16:51,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7480a22614804c0b83edea2caf80a752 2024-11-18T03:16:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T03:16:51,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899871525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T03:16:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:51,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899871555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899871556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899871557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899871560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/787f18e0409e485a8e0e2e24c508cf77 is 50, key is test_row_0/C:col10/1731899811214/Put/seqid=0 2024-11-18T03:16:51,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742305_1481 (size=12001) 2024-11-18T03:16:51,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899871663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899871681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899871681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899871681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899871696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T03:16:51,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T03:16:51,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T03:16:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:51,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:51,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899871892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899871901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899871901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899871901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:51,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899871911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/787f18e0409e485a8e0e2e24c508cf77 2024-11-18T03:16:52,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T03:16:52,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:52,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:52,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:52,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/d15272a30aeb427fb9a3a67306c2d622 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d15272a30aeb427fb9a3a67306c2d622 2024-11-18T03:16:52,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d15272a30aeb427fb9a3a67306c2d622, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T03:16:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7480a22614804c0b83edea2caf80a752 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7480a22614804c0b83edea2caf80a752 2024-11-18T03:16:52,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7480a22614804c0b83edea2caf80a752, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T03:16:52,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/787f18e0409e485a8e0e2e24c508cf77 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/787f18e0409e485a8e0e2e24c508cf77 2024-11-18T03:16:52,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/787f18e0409e485a8e0e2e24c508cf77, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T03:16:52,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 9d402698952bfb73679bd006311c3148 in 938ms, sequenceid=12, compaction requested=false 2024-11-18T03:16:52,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:52,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T03:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:52,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T03:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:16:52,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/35de357400244c6797afed84dcd023f3 is 50, key is test_row_0/A:col10/1731899811554/Put/seqid=0 2024-11-18T03:16:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:52,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742306_1482 (size=12001) 2024-11-18T03:16:52,261 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/35de357400244c6797afed84dcd023f3 2024-11-18T03:16:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899872244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899872245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899872266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899872271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899872276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/5994f4c46d54428b8df0f2a8c8eb4455 is 50, key is test_row_0/B:col10/1731899811554/Put/seqid=0 2024-11-18T03:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T03:16:52,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742307_1483 (size=12001) 2024-11-18T03:16:52,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/5994f4c46d54428b8df0f2a8c8eb4455 2024-11-18T03:16:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/1b15d95439234d0aa2f3be16e384816e is 50, key is test_row_0/C:col10/1731899811554/Put/seqid=0 2024-11-18T03:16:52,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899872387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899872381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899872411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742308_1484 (size=12001) 2024-11-18T03:16:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899872412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899872412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899872642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899872642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899872642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899872642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899872642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:52,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/1b15d95439234d0aa2f3be16e384816e 2024-11-18T03:16:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/35de357400244c6797afed84dcd023f3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/35de357400244c6797afed84dcd023f3 2024-11-18T03:16:52,863 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/35de357400244c6797afed84dcd023f3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T03:16:52,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/5994f4c46d54428b8df0f2a8c8eb4455 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5994f4c46d54428b8df0f2a8c8eb4455 2024-11-18T03:16:52,869 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5994f4c46d54428b8df0f2a8c8eb4455, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T03:16:52,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/1b15d95439234d0aa2f3be16e384816e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/1b15d95439234d0aa2f3be16e384816e 2024-11-18T03:16:52,892 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/1b15d95439234d0aa2f3be16e384816e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T03:16:52,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 9d402698952bfb73679bd006311c3148 in 699ms, sequenceid=49, compaction requested=false 2024-11-18T03:16:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-18T03:16:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-18T03:16:52,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-18T03:16:52,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6820 sec 2024-11-18T03:16:52,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7100 sec 2024-11-18T03:16:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:53,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:16:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:53,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ad7fecc4755e4c0f8f4cddf40cfc67a3 is 50, key is test_row_0/A:col10/1731899812268/Put/seqid=0 2024-11-18T03:16:53,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742309_1485 (size=19021) 2024-11-18T03:16:53,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ad7fecc4755e4c0f8f4cddf40cfc67a3 2024-11-18T03:16:53,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899873105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899873105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899873137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899873137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899873141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899873243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/169c1be4a488482f9151652780309373 is 50, key is test_row_0/B:col10/1731899812268/Put/seqid=0 2024-11-18T03:16:53,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899873260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899873281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899873282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899873282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742310_1486 (size=12001) 2024-11-18T03:16:53,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/169c1be4a488482f9151652780309373 2024-11-18T03:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T03:16:53,385 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:53,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-18T03:16:53,401 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T03:16:53,402 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:53,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:53,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/354c2e8a6743416ba2209d45e7e74d95 is 50, key is test_row_0/C:col10/1731899812268/Put/seqid=0 2024-11-18T03:16:53,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742311_1487 (size=12001) 2024-11-18T03:16:53,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/354c2e8a6743416ba2209d45e7e74d95 2024-11-18T03:16:53,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899873471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T03:16:53,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899873502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899873511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899873513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899873513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ad7fecc4755e4c0f8f4cddf40cfc67a3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ad7fecc4755e4c0f8f4cddf40cfc67a3 2024-11-18T03:16:53,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T03:16:53,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:53,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:53,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:53,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:53,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:53,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ad7fecc4755e4c0f8f4cddf40cfc67a3, entries=300, sequenceid=63, filesize=18.6 K 2024-11-18T03:16:53,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/169c1be4a488482f9151652780309373 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/169c1be4a488482f9151652780309373 2024-11-18T03:16:53,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/169c1be4a488482f9151652780309373, entries=150, sequenceid=63, filesize=11.7 K 2024-11-18T03:16:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/354c2e8a6743416ba2209d45e7e74d95 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/354c2e8a6743416ba2209d45e7e74d95 2024-11-18T03:16:53,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/354c2e8a6743416ba2209d45e7e74d95, entries=150, sequenceid=63, filesize=11.7 K 2024-11-18T03:16:53,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9d402698952bfb73679bd006311c3148 in 666ms, sequenceid=63, compaction requested=true 2024-11-18T03:16:53,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:53,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:53,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:53,675 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:53,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:53,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:53,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:53,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:16:53,679 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:53,687 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:53,688 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:16:53,688 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:53,688 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d15272a30aeb427fb9a3a67306c2d622, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/35de357400244c6797afed84dcd023f3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ad7fecc4755e4c0f8f4cddf40cfc67a3] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=42.0 K 2024-11-18T03:16:53,693 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d15272a30aeb427fb9a3a67306c2d622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731899811209 2024-11-18T03:16:53,703 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35de357400244c6797afed84dcd023f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731899811518 2024-11-18T03:16:53,704 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7fecc4755e4c0f8f4cddf40cfc67a3, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899812238 2024-11-18T03:16:53,707 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:53,707 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:16:53,707 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:53,708 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7480a22614804c0b83edea2caf80a752, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5994f4c46d54428b8df0f2a8c8eb4455, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/169c1be4a488482f9151652780309373] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.2 K 2024-11-18T03:16:53,712 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7480a22614804c0b83edea2caf80a752, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731899811209 2024-11-18T03:16:53,717 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5994f4c46d54428b8df0f2a8c8eb4455, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731899811518 2024-11-18T03:16:53,720 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 169c1be4a488482f9151652780309373, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899812264 2024-11-18T03:16:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T03:16:53,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T03:16:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:53,728 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:16:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:53,739 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:53,739 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/e078efde167b44218ed1ce66e7ffb99a is 50, key is test_row_0/A:col10/1731899812268/Put/seqid=0 2024-11-18T03:16:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/8f4982741f8d43edb380d1afa1351217 is 50, key is test_row_0/A:col10/1731899813136/Put/seqid=0 2024-11-18T03:16:53,844 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:53,844 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/97247aabf2ba444699a7d460e8339c1f is 50, key is test_row_0/B:col10/1731899812268/Put/seqid=0 2024-11-18T03:16:53,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742312_1488 (size=12104) 2024-11-18T03:16:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:53,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742313_1489 (size=12001) 2024-11-18T03:16:53,884 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/8f4982741f8d43edb380d1afa1351217 2024-11-18T03:16:53,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742314_1490 (size=12104) 2024-11-18T03:16:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/f1a37115bc60467ea96bc86943064cfc is 50, key is test_row_0/B:col10/1731899813136/Put/seqid=0 2024-11-18T03:16:53,913 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/e078efde167b44218ed1ce66e7ffb99a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e078efde167b44218ed1ce66e7ffb99a 2024-11-18T03:16:53,915 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/97247aabf2ba444699a7d460e8339c1f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/97247aabf2ba444699a7d460e8339c1f 2024-11-18T03:16:53,921 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into e078efde167b44218ed1ce66e7ffb99a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:53,921 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:53,921 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899813674; duration=0sec 2024-11-18T03:16:53,921 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:53,921 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:16:53,921 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:53,928 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:53,928 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:16:53,928 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:53,928 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/787f18e0409e485a8e0e2e24c508cf77, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/1b15d95439234d0aa2f3be16e384816e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/354c2e8a6743416ba2209d45e7e74d95] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.2 K 2024-11-18T03:16:53,930 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787f18e0409e485a8e0e2e24c508cf77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731899811209 2024-11-18T03:16:53,932 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b15d95439234d0aa2f3be16e384816e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731899811518 2024-11-18T03:16:53,933 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into 97247aabf2ba444699a7d460e8339c1f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:53,933 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:53,933 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899813675; duration=0sec 2024-11-18T03:16:53,933 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:53,933 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:16:53,933 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 354c2e8a6743416ba2209d45e7e74d95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899812264 2024-11-18T03:16:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899873895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:53,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742315_1491 (size=12001) 2024-11-18T03:16:53,974 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:53,975 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/cd30d6b7c7034fd39e400eed69a87586 is 50, key is test_row_0/C:col10/1731899812268/Put/seqid=0 2024-11-18T03:16:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899873944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899873948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899873949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899873949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T03:16:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742316_1492 (size=12104) 2024-11-18T03:16:54,065 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/cd30d6b7c7034fd39e400eed69a87586 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd30d6b7c7034fd39e400eed69a87586 2024-11-18T03:16:54,081 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into cd30d6b7c7034fd39e400eed69a87586(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:54,081 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:54,081 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899813675; duration=0sec 2024-11-18T03:16:54,081 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:54,081 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:16:54,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899874051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,122 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:16:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899874121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899874132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899874132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899874134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899874291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899874352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,379 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/f1a37115bc60467ea96bc86943064cfc 2024-11-18T03:16:54,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899874367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899874367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899874367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/cd941ec2765c45aba829008ab6b6d9c9 is 50, key is test_row_0/C:col10/1731899813136/Put/seqid=0 2024-11-18T03:16:54,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742317_1493 (size=12001) 2024-11-18T03:16:54,440 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/cd941ec2765c45aba829008ab6b6d9c9 2024-11-18T03:16:54,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/8f4982741f8d43edb380d1afa1351217 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/8f4982741f8d43edb380d1afa1351217 2024-11-18T03:16:54,502 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/8f4982741f8d43edb380d1afa1351217, entries=150, sequenceid=81, filesize=11.7 K 2024-11-18T03:16:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/f1a37115bc60467ea96bc86943064cfc as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f1a37115bc60467ea96bc86943064cfc 2024-11-18T03:16:54,518 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f1a37115bc60467ea96bc86943064cfc, entries=150, sequenceid=81, filesize=11.7 K 2024-11-18T03:16:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/cd941ec2765c45aba829008ab6b6d9c9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd941ec2765c45aba829008ab6b6d9c9 2024-11-18T03:16:54,530 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd941ec2765c45aba829008ab6b6d9c9, entries=150, sequenceid=81, filesize=11.7 K 2024-11-18T03:16:54,531 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9d402698952bfb73679bd006311c3148 in 803ms, sequenceid=81, compaction requested=false 2024-11-18T03:16:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-18T03:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-18T03:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T03:16:54,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-18T03:16:54,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1360 sec 2024-11-18T03:16:54,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.1750 sec 2024-11-18T03:16:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:54,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:16:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:54,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/2358409e02454d7ab000452fe12c0f7c is 50, key is test_row_0/A:col10/1731899814638/Put/seqid=0 2024-11-18T03:16:54,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742318_1494 (size=14341) 2024-11-18T03:16:54,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/2358409e02454d7ab000452fe12c0f7c 2024-11-18T03:16:54,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899874763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899874766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899874770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/9c4f10f7fe1b4b2babe491c2cffc47b4 is 50, key is test_row_0/B:col10/1731899814638/Put/seqid=0 2024-11-18T03:16:54,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899874802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899874812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742319_1495 (size=12001) 2024-11-18T03:16:54,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899874912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899874912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899874921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899874964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899874964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899875141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899875141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899875142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899875193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899875193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/9c4f10f7fe1b4b2babe491c2cffc47b4 2024-11-18T03:16:55,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/6274b695631049b3b73254d4a3e4df20 is 50, key is test_row_0/C:col10/1731899814638/Put/seqid=0 2024-11-18T03:16:55,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742320_1496 (size=12001) 2024-11-18T03:16:55,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899875461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899875462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899875462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899875521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899875520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T03:16:55,553 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:55,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-18T03:16:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T03:16:55,569 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:55,569 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:55,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T03:16:55,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T03:16:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:55,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:55,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/6274b695631049b3b73254d4a3e4df20 2024-11-18T03:16:55,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/2358409e02454d7ab000452fe12c0f7c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2358409e02454d7ab000452fe12c0f7c 2024-11-18T03:16:55,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2358409e02454d7ab000452fe12c0f7c, entries=200, sequenceid=99, filesize=14.0 K 2024-11-18T03:16:55,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/9c4f10f7fe1b4b2babe491c2cffc47b4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9c4f10f7fe1b4b2babe491c2cffc47b4 2024-11-18T03:16:55,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9c4f10f7fe1b4b2babe491c2cffc47b4, entries=150, sequenceid=99, filesize=11.7 K 2024-11-18T03:16:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/6274b695631049b3b73254d4a3e4df20 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/6274b695631049b3b73254d4a3e4df20 2024-11-18T03:16:55,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/6274b695631049b3b73254d4a3e4df20, entries=150, sequenceid=99, filesize=11.7 K 2024-11-18T03:16:55,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9d402698952bfb73679bd006311c3148 in 1230ms, sequenceid=99, compaction requested=true 2024-11-18T03:16:55,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:55,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:55,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:55,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:55,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:55,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:55,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:16:55,879 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:55,879 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T03:16:55,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T03:16:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:55,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:16:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:55,889 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:55,889 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:16:55,889 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:55,890 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/97247aabf2ba444699a7d460e8339c1f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f1a37115bc60467ea96bc86943064cfc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9c4f10f7fe1b4b2babe491c2cffc47b4] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.3 K 2024-11-18T03:16:55,894 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:55,894 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:16:55,894 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:55,894 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e078efde167b44218ed1ce66e7ffb99a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/8f4982741f8d43edb380d1afa1351217, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2358409e02454d7ab000452fe12c0f7c] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=37.5 K 2024-11-18T03:16:55,895 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 97247aabf2ba444699a7d460e8339c1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899812264 2024-11-18T03:16:55,899 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f1a37115bc60467ea96bc86943064cfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731899813084 2024-11-18T03:16:55,899 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e078efde167b44218ed1ce66e7ffb99a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899812264 2024-11-18T03:16:55,902 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4982741f8d43edb380d1afa1351217, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731899813084 2024-11-18T03:16:55,902 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4f10f7fe1b4b2babe491c2cffc47b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731899813941 2024-11-18T03:16:55,907 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2358409e02454d7ab000452fe12c0f7c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731899813941 2024-11-18T03:16:55,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/7220092a4f584bec81120ef0cf5b80ea is 50, key is test_row_0/A:col10/1731899814751/Put/seqid=0 2024-11-18T03:16:55,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742321_1497 (size=12001) 2024-11-18T03:16:55,952 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/7220092a4f584bec81120ef0cf5b80ea 2024-11-18T03:16:55,952 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:55,952 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/42b12ef709334691be68848204b73743 is 50, key is test_row_0/A:col10/1731899814638/Put/seqid=0 2024-11-18T03:16:55,982 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:55,983 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/567c072ab10e468097271508cbab4f38 is 50, key is test_row_0/B:col10/1731899814638/Put/seqid=0 2024-11-18T03:16:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/289376415cb8469e822156c08d34d11b is 50, key is test_row_0/B:col10/1731899814751/Put/seqid=0 2024-11-18T03:16:56,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:56,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742322_1498 (size=12207) 2024-11-18T03:16:56,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742324_1500 (size=12001) 2024-11-18T03:16:56,092 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/42b12ef709334691be68848204b73743 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/42b12ef709334691be68848204b73743 2024-11-18T03:16:56,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742323_1499 (size=12207) 2024-11-18T03:16:56,143 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into 42b12ef709334691be68848204b73743(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:56,143 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:56,143 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899815878; duration=0sec 2024-11-18T03:16:56,143 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:56,143 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:16:56,143 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:56,146 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:56,146 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:16:56,146 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:56,146 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd30d6b7c7034fd39e400eed69a87586, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd941ec2765c45aba829008ab6b6d9c9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/6274b695631049b3b73254d4a3e4df20] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.3 K 2024-11-18T03:16:56,147 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd30d6b7c7034fd39e400eed69a87586, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899812264 2024-11-18T03:16:56,147 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd941ec2765c45aba829008ab6b6d9c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731899813084 2024-11-18T03:16:56,148 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6274b695631049b3b73254d4a3e4df20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731899813941 2024-11-18T03:16:56,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899876127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899876130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899876132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,189 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#418 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T03:16:56,196 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ceaec43ca4fa4720915ed56aab5a8484 is 50, key is test_row_0/C:col10/1731899814638/Put/seqid=0 2024-11-18T03:16:56,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899876139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899876171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742325_1501 (size=12207) 2024-11-18T03:16:56,269 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ceaec43ca4fa4720915ed56aab5a8484 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ceaec43ca4fa4720915ed56aab5a8484 2024-11-18T03:16:56,284 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into ceaec43ca4fa4720915ed56aab5a8484(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:56,285 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:56,285 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899815879; duration=0sec 2024-11-18T03:16:56,285 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:56,285 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:16:56,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899876285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899876294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899876296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899876308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899876327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,477 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/289376415cb8469e822156c08d34d11b 2024-11-18T03:16:56,510 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/567c072ab10e468097271508cbab4f38 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/567c072ab10e468097271508cbab4f38 2024-11-18T03:16:56,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/11381eaf3f904b64a8ca55b867add888 is 50, key is test_row_0/C:col10/1731899814751/Put/seqid=0 2024-11-18T03:16:56,523 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into 567c072ab10e468097271508cbab4f38(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:56,523 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:56,523 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899815879; duration=0sec 2024-11-18T03:16:56,523 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:56,523 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:16:56,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899876532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899876532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742326_1502 (size=12001) 2024-11-18T03:16:56,567 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/11381eaf3f904b64a8ca55b867add888 2024-11-18T03:16:56,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/7220092a4f584bec81120ef0cf5b80ea as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7220092a4f584bec81120ef0cf5b80ea 2024-11-18T03:16:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899876545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899876552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,587 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7220092a4f584bec81120ef0cf5b80ea, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T03:16:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899876567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/289376415cb8469e822156c08d34d11b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/289376415cb8469e822156c08d34d11b 2024-11-18T03:16:56,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/289376415cb8469e822156c08d34d11b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T03:16:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/11381eaf3f904b64a8ca55b867add888 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/11381eaf3f904b64a8ca55b867add888 2024-11-18T03:16:56,612 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/11381eaf3f904b64a8ca55b867add888, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T03:16:56,613 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9d402698952bfb73679bd006311c3148 in 724ms, sequenceid=116, compaction requested=false 2024-11-18T03:16:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-18T03:16:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-18T03:16:56,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-18T03:16:56,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-11-18T03:16:56,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0710 sec 2024-11-18T03:16:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T03:16:56,709 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:56,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-18T03:16:56,715 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:56,716 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:56,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T03:16:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T03:16:56,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T03:16:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:56,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T03:16:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:56,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:56,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c1de0445bb6d4529bb301529164a7346 is 50, key is test_row_0/A:col10/1731899816129/Put/seqid=0 2024-11-18T03:16:56,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742327_1503 (size=14491) 2024-11-18T03:16:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899876924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899876926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899876928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899876941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899876943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T03:16:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T03:16:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899877061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899877061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899877061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899877062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899877071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T03:16:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899877291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c1de0445bb6d4529bb301529164a7346 2024-11-18T03:16:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899877301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899877302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899877307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899877307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b6baab0b2af94522862d4e80150c47f1 is 50, key is test_row_0/B:col10/1731899816129/Put/seqid=0 2024-11-18T03:16:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T03:16:57,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T03:16:57,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:57,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:57,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742328_1504 (size=12101) 2024-11-18T03:16:57,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b6baab0b2af94522862d4e80150c47f1 2024-11-18T03:16:57,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/db9517a620534f9aa44ffe76d89410eb is 50, key is test_row_0/C:col10/1731899816129/Put/seqid=0 2024-11-18T03:16:57,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742329_1505 (size=12101) 2024-11-18T03:16:57,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/db9517a620534f9aa44ffe76d89410eb 2024-11-18T03:16:57,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c1de0445bb6d4529bb301529164a7346 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c1de0445bb6d4529bb301529164a7346 2024-11-18T03:16:57,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c1de0445bb6d4529bb301529164a7346, entries=200, sequenceid=135, filesize=14.2 K 2024-11-18T03:16:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b6baab0b2af94522862d4e80150c47f1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6baab0b2af94522862d4e80150c47f1 2024-11-18T03:16:57,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6baab0b2af94522862d4e80150c47f1, entries=150, sequenceid=135, filesize=11.8 K 2024-11-18T03:16:57,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/db9517a620534f9aa44ffe76d89410eb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/db9517a620534f9aa44ffe76d89410eb 2024-11-18T03:16:57,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/db9517a620534f9aa44ffe76d89410eb, entries=150, sequenceid=135, filesize=11.8 K 2024-11-18T03:16:57,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9d402698952bfb73679bd006311c3148 in 628ms, sequenceid=135, compaction requested=true 2024-11-18T03:16:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:16:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T03:16:57,499 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:57,499 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:57,503 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:57,503 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:16:57,503 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,503 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ceaec43ca4fa4720915ed56aab5a8484, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/11381eaf3f904b64a8ca55b867add888, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/db9517a620534f9aa44ffe76d89410eb] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.5 K 2024-11-18T03:16:57,503 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:57,503 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:16:57,503 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,503 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/42b12ef709334691be68848204b73743, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7220092a4f584bec81120ef0cf5b80ea, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c1de0445bb6d4529bb301529164a7346] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=37.8 K 2024-11-18T03:16:57,508 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b12ef709334691be68848204b73743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731899813941 2024-11-18T03:16:57,508 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ceaec43ca4fa4720915ed56aab5a8484, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731899813941 2024-11-18T03:16:57,513 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 11381eaf3f904b64a8ca55b867add888, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731899814751 2024-11-18T03:16:57,513 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7220092a4f584bec81120ef0cf5b80ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731899814751 2024-11-18T03:16:57,513 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting db9517a620534f9aa44ffe76d89410eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731899816128 2024-11-18T03:16:57,518 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1de0445bb6d4529bb301529164a7346, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731899816128 2024-11-18T03:16:57,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T03:16:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,521 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T03:16:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/92ea590ddfa44c8f9f97769bf4179a08 is 50, key is test_row_0/A:col10/1731899816939/Put/seqid=0 2024-11-18T03:16:57,574 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:57,575 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/2c9acecf5d4d44f7b56edd886f75dc48 is 50, key is test_row_0/A:col10/1731899816129/Put/seqid=0 2024-11-18T03:16:57,582 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:57,583 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/7225dc2523664fb39b2d2f2cee1b32ee is 50, key is test_row_0/C:col10/1731899816129/Put/seqid=0 2024-11-18T03:16:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:57,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:57,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742330_1506 (size=12151) 2024-11-18T03:16:57,652 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/92ea590ddfa44c8f9f97769bf4179a08 2024-11-18T03:16:57,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742331_1507 (size=12409) 2024-11-18T03:16:57,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742332_1508 (size=12409) 2024-11-18T03:16:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/505bb7114be5416bbb80ef24db8469b5 is 50, key is test_row_0/B:col10/1731899816939/Put/seqid=0 2024-11-18T03:16:57,698 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/7225dc2523664fb39b2d2f2cee1b32ee as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/7225dc2523664fb39b2d2f2cee1b32ee 2024-11-18T03:16:57,726 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into 7225dc2523664fb39b2d2f2cee1b32ee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:57,726 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:57,726 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899817497; duration=0sec 2024-11-18T03:16:57,726 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:57,726 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:16:57,726 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:57,728 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:57,728 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:16:57,728 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:57,729 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/567c072ab10e468097271508cbab4f38, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/289376415cb8469e822156c08d34d11b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6baab0b2af94522862d4e80150c47f1] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.5 K 2024-11-18T03:16:57,730 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 567c072ab10e468097271508cbab4f38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731899813941 2024-11-18T03:16:57,730 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 289376415cb8469e822156c08d34d11b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731899814751 2024-11-18T03:16:57,736 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b6baab0b2af94522862d4e80150c47f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731899816128 2024-11-18T03:16:57,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742333_1509 (size=12151) 2024-11-18T03:16:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899877727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899877728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899877731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899877734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,771 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,772 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/a7b620ee52e243b5bf2aa695eaf21ae3 is 50, key is test_row_0/B:col10/1731899816129/Put/seqid=0 2024-11-18T03:16:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899877749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742334_1510 (size=12409) 2024-11-18T03:16:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T03:16:57,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899877871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899877881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899877882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899877884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:57,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899877909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,080 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/2c9acecf5d4d44f7b56edd886f75dc48 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2c9acecf5d4d44f7b56edd886f75dc48 2024-11-18T03:16:58,101 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into 2c9acecf5d4d44f7b56edd886f75dc48(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:58,101 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:58,101 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899817497; duration=0sec 2024-11-18T03:16:58,101 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:58,101 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:16:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899878101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899878101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899878101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899878121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/505bb7114be5416bbb80ef24db8469b5 2024-11-18T03:16:58,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899878144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,220 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/a7b620ee52e243b5bf2aa695eaf21ae3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/a7b620ee52e243b5bf2aa695eaf21ae3 2024-11-18T03:16:58,233 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into a7b620ee52e243b5bf2aa695eaf21ae3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:58,233 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:58,233 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899817497; duration=0sec 2024-11-18T03:16:58,233 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:58,233 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:16:58,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/09c6d0856f7449809d0bc5808556696a is 50, key is test_row_0/C:col10/1731899816939/Put/seqid=0 2024-11-18T03:16:58,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742335_1511 (size=12151) 2024-11-18T03:16:58,310 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/09c6d0856f7449809d0bc5808556696a 2024-11-18T03:16:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/92ea590ddfa44c8f9f97769bf4179a08 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/92ea590ddfa44c8f9f97769bf4179a08 2024-11-18T03:16:58,319 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/92ea590ddfa44c8f9f97769bf4179a08, entries=150, sequenceid=151, filesize=11.9 K 2024-11-18T03:16:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/505bb7114be5416bbb80ef24db8469b5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/505bb7114be5416bbb80ef24db8469b5 2024-11-18T03:16:58,330 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/505bb7114be5416bbb80ef24db8469b5, entries=150, sequenceid=151, filesize=11.9 K 2024-11-18T03:16:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/09c6d0856f7449809d0bc5808556696a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/09c6d0856f7449809d0bc5808556696a 2024-11-18T03:16:58,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/09c6d0856f7449809d0bc5808556696a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-18T03:16:58,336 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9d402698952bfb73679bd006311c3148 in 816ms, sequenceid=151, compaction requested=false 2024-11-18T03:16:58,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:58,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:58,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-18T03:16:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-18T03:16:58,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-18T03:16:58,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-11-18T03:16:58,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6340 sec 2024-11-18T03:16:58,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:16:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:58,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/84fe4196fcaa42f0a8f102596955cdb4 is 50, key is test_row_0/A:col10/1731899817731/Put/seqid=0 2024-11-18T03:16:58,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742336_1512 (size=14541) 2024-11-18T03:16:58,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899878566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899878595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899878597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899878599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899878600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899878712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899878745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899878745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899878745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899878745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T03:16:58,873 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:16:58,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:16:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-18T03:16:58,875 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:16:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T03:16:58,878 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:16:58,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:16:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899878941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:58,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/84fe4196fcaa42f0a8f102596955cdb4 2024-11-18T03:16:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T03:16:58,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/00e78d2b0c634d73950bbc600a2cb0e7 is 50, key is test_row_0/B:col10/1731899817731/Put/seqid=0 2024-11-18T03:16:59,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899878973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899878974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899878974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899878974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T03:16:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:59,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:59,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742337_1513 (size=12151) 2024-11-18T03:16:59,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/00e78d2b0c634d73950bbc600a2cb0e7 2024-11-18T03:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:16:59,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9727845528ff438fa1c3c49596dc4e92 is 50, key is test_row_0/C:col10/1731899817731/Put/seqid=0 2024-11-18T03:16:59,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742338_1514 (size=12151) 2024-11-18T03:16:59,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9727845528ff438fa1c3c49596dc4e92 2024-11-18T03:16:59,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/84fe4196fcaa42f0a8f102596955cdb4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/84fe4196fcaa42f0a8f102596955cdb4 2024-11-18T03:16:59,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/84fe4196fcaa42f0a8f102596955cdb4, entries=200, sequenceid=171, filesize=14.2 K 2024-11-18T03:16:59,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/00e78d2b0c634d73950bbc600a2cb0e7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/00e78d2b0c634d73950bbc600a2cb0e7 2024-11-18T03:16:59,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/00e78d2b0c634d73950bbc600a2cb0e7, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T03:16:59,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9727845528ff438fa1c3c49596dc4e92 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9727845528ff438fa1c3c49596dc4e92 2024-11-18T03:16:59,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9727845528ff438fa1c3c49596dc4e92, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T03:16:59,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9d402698952bfb73679bd006311c3148 in 699ms, sequenceid=171, compaction requested=true 2024-11-18T03:16:59,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:59,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:16:59,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:59,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:16:59,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:16:59,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:16:59,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T03:16:59,162 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:59,164 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:59,164 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:59,164 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:16:59,164 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:59,164 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2c9acecf5d4d44f7b56edd886f75dc48, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/92ea590ddfa44c8f9f97769bf4179a08, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/84fe4196fcaa42f0a8f102596955cdb4] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=38.2 K 2024-11-18T03:16:59,165 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c9acecf5d4d44f7b56edd886f75dc48, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731899816128 2024-11-18T03:16:59,165 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92ea590ddfa44c8f9f97769bf4179a08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731899816924 2024-11-18T03:16:59,165 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84fe4196fcaa42f0a8f102596955cdb4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731899817678 2024-11-18T03:16:59,175 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:59,175 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:16:59,175 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:59,175 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/7225dc2523664fb39b2d2f2cee1b32ee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/09c6d0856f7449809d0bc5808556696a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9727845528ff438fa1c3c49596dc4e92] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.9 K 2024-11-18T03:16:59,176 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7225dc2523664fb39b2d2f2cee1b32ee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731899816128 2024-11-18T03:16:59,178 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c6d0856f7449809d0bc5808556696a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731899816924 2024-11-18T03:16:59,181 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9727845528ff438fa1c3c49596dc4e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731899817678 2024-11-18T03:16:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T03:16:59,206 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:59,207 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/6f60670d40bf4ae9ae2d342e5ea96bfe is 50, key is test_row_0/A:col10/1731899817731/Put/seqid=0 2024-11-18T03:16:59,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T03:16:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:59,210 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:16:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:16:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:16:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:16:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:16:59,255 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:59,256 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/0a9104df59d04a37918377545f7fe91e is 50, key is test_row_0/C:col10/1731899817731/Put/seqid=0 2024-11-18T03:16:59,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/9db474b8a2154c828888133124b05ade is 50, key is test_row_0/A:col10/1731899818598/Put/seqid=0 2024-11-18T03:16:59,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742339_1515 (size=12561) 2024-11-18T03:16:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:16:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:16:59,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742340_1516 (size=12561) 2024-11-18T03:16:59,320 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/6f60670d40bf4ae9ae2d342e5ea96bfe as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6f60670d40bf4ae9ae2d342e5ea96bfe 2024-11-18T03:16:59,331 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into 6f60670d40bf4ae9ae2d342e5ea96bfe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:59,331 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:59,331 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899819161; duration=0sec 2024-11-18T03:16:59,331 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:16:59,331 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:16:59,331 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:16:59,341 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:16:59,341 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:16:59,341 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:16:59,341 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/a7b620ee52e243b5bf2aa695eaf21ae3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/505bb7114be5416bbb80ef24db8469b5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/00e78d2b0c634d73950bbc600a2cb0e7] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=35.9 K 2024-11-18T03:16:59,342 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7b620ee52e243b5bf2aa695eaf21ae3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731899816128 2024-11-18T03:16:59,342 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 505bb7114be5416bbb80ef24db8469b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731899816924 2024-11-18T03:16:59,342 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e78d2b0c634d73950bbc600a2cb0e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731899817678 2024-11-18T03:16:59,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742341_1517 (size=12151) 2024-11-18T03:16:59,388 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:16:59,388 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/fcb4be2a8a9c483fb5c5061d6a439bb0 is 50, key is test_row_0/B:col10/1731899817731/Put/seqid=0 2024-11-18T03:16:59,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742342_1518 (size=12561) 2024-11-18T03:16:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899879415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899879416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,484 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/fcb4be2a8a9c483fb5c5061d6a439bb0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fcb4be2a8a9c483fb5c5061d6a439bb0 2024-11-18T03:16:59,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899879454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899879459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T03:16:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899879466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,534 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into fcb4be2a8a9c483fb5c5061d6a439bb0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:59,535 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:59,535 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899819162; duration=0sec 2024-11-18T03:16:59,535 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:59,535 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:16:59,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899879572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899879572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899879604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899879612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899879612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,767 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/0a9104df59d04a37918377545f7fe91e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0a9104df59d04a37918377545f7fe91e 2024-11-18T03:16:59,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/9db474b8a2154c828888133124b05ade 2024-11-18T03:16:59,793 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into 0a9104df59d04a37918377545f7fe91e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:16:59,793 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:16:59,793 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899819162; duration=0sec 2024-11-18T03:16:59,793 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:16:59,793 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:16:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7e722a22ed6245cab8d4a2d618238d64 is 50, key is test_row_0/B:col10/1731899818598/Put/seqid=0 2024-11-18T03:16:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899879808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899879812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899879831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899879836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:16:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899879837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:16:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:16:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:16:59,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742343_1519 (size=12151) 2024-11-18T03:16:59,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7e722a22ed6245cab8d4a2d618238d64 2024-11-18T03:16:59,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9ebcff4a49ae49b989a88c7b09935a93 is 50, key is test_row_0/C:col10/1731899818598/Put/seqid=0 2024-11-18T03:16:59,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742344_1520 (size=12151) 2024-11-18T03:16:59,963 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9ebcff4a49ae49b989a88c7b09935a93 2024-11-18T03:16:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/9db474b8a2154c828888133124b05ade as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/9db474b8a2154c828888133124b05ade 2024-11-18T03:16:59,995 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/9db474b8a2154c828888133124b05ade, entries=150, sequenceid=186, filesize=11.9 K 2024-11-18T03:17:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7e722a22ed6245cab8d4a2d618238d64 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7e722a22ed6245cab8d4a2d618238d64 2024-11-18T03:17:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T03:17:00,020 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7e722a22ed6245cab8d4a2d618238d64, entries=150, sequenceid=186, filesize=11.9 K 2024-11-18T03:17:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9ebcff4a49ae49b989a88c7b09935a93 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ebcff4a49ae49b989a88c7b09935a93 2024-11-18T03:17:00,055 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ebcff4a49ae49b989a88c7b09935a93, entries=150, sequenceid=186, filesize=11.9 K 2024-11-18T03:17:00,056 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 9d402698952bfb73679bd006311c3148 in 846ms, sequenceid=186, compaction requested=false 2024-11-18T03:17:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-18T03:17:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-18T03:17:00,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-18T03:17:00,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1850 sec 2024-11-18T03:17:00,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.1960 sec 2024-11-18T03:17:00,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T03:17:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:00,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/4f29622273dc4747a1b420f7217387b7 is 50, key is test_row_0/A:col10/1731899820168/Put/seqid=0 2024-11-18T03:17:00,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742345_1521 (size=14541) 2024-11-18T03:17:00,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/4f29622273dc4747a1b420f7217387b7 2024-11-18T03:17:00,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7b5f01fddbf24e119592e53220699c47 is 50, key is test_row_0/B:col10/1731899820168/Put/seqid=0 2024-11-18T03:17:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899880222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899880232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899880221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899880271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899880273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742346_1522 (size=12151) 2024-11-18T03:17:00,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899880389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899880389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899880389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899880401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899880421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899880622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899880622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899880622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899880622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899880641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7b5f01fddbf24e119592e53220699c47 2024-11-18T03:17:00,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/090e80391ddc4c2187aaedb5a4fe3b29 is 50, key is test_row_0/C:col10/1731899820168/Put/seqid=0 2024-11-18T03:17:00,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742347_1523 (size=12151) 2024-11-18T03:17:00,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/090e80391ddc4c2187aaedb5a4fe3b29 2024-11-18T03:17:00,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/4f29622273dc4747a1b420f7217387b7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/4f29622273dc4747a1b420f7217387b7 2024-11-18T03:17:00,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/4f29622273dc4747a1b420f7217387b7, entries=200, sequenceid=208, filesize=14.2 K 2024-11-18T03:17:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/7b5f01fddbf24e119592e53220699c47 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7b5f01fddbf24e119592e53220699c47 2024-11-18T03:17:00,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7b5f01fddbf24e119592e53220699c47, entries=150, sequenceid=208, filesize=11.9 K 2024-11-18T03:17:00,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899880953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899880953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899880953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899880956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/090e80391ddc4c2187aaedb5a4fe3b29 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/090e80391ddc4c2187aaedb5a4fe3b29 2024-11-18T03:17:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:00,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899880971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/090e80391ddc4c2187aaedb5a4fe3b29, entries=150, sequenceid=208, filesize=11.9 K 2024-11-18T03:17:01,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 9d402698952bfb73679bd006311c3148 in 839ms, sequenceid=208, compaction requested=true 2024-11-18T03:17:01,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:01,010 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:01,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:01,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:01,011 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:01,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:01,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:01,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:01,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:01,016 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:01,017 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:17:01,017 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:01,017 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6f60670d40bf4ae9ae2d342e5ea96bfe, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/9db474b8a2154c828888133124b05ade, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/4f29622273dc4747a1b420f7217387b7] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=38.3 K 2024-11-18T03:17:01,017 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:01,017 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:17:01,017 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:01,017 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fcb4be2a8a9c483fb5c5061d6a439bb0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7e722a22ed6245cab8d4a2d618238d64, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7b5f01fddbf24e119592e53220699c47] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.0 K 2024-11-18T03:17:01,022 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb4be2a8a9c483fb5c5061d6a439bb0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731899817678 2024-11-18T03:17:01,022 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f60670d40bf4ae9ae2d342e5ea96bfe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731899817678 2024-11-18T03:17:01,024 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e722a22ed6245cab8d4a2d618238d64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731899818503 2024-11-18T03:17:01,024 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db474b8a2154c828888133124b05ade, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731899818503 2024-11-18T03:17:01,025 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f29622273dc4747a1b420f7217387b7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731899819448 2024-11-18T03:17:01,026 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b5f01fddbf24e119592e53220699c47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731899819451 2024-11-18T03:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T03:17:01,036 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:01,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-18T03:17:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T03:17:01,046 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:01,046 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:01,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:01,049 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:01,049 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/a9a9f9cad1154a33be2a961b1b3e38f8 is 50, key is test_row_0/A:col10/1731899820168/Put/seqid=0 2024-11-18T03:17:01,069 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:01,070 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/f081289136f6414a9305961438aad1a3 is 50, key is test_row_0/B:col10/1731899820168/Put/seqid=0 2024-11-18T03:17:01,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742348_1524 (size=12663) 2024-11-18T03:17:01,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742349_1525 (size=12663) 2024-11-18T03:17:01,136 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/a9a9f9cad1154a33be2a961b1b3e38f8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/a9a9f9cad1154a33be2a961b1b3e38f8 2024-11-18T03:17:01,140 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/f081289136f6414a9305961438aad1a3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f081289136f6414a9305961438aad1a3 2024-11-18T03:17:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T03:17:01,155 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into f081289136f6414a9305961438aad1a3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:01,155 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:01,155 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899821011; duration=0sec 2024-11-18T03:17:01,155 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:01,156 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:17:01,156 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:01,163 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:01,163 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:17:01,164 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:01,164 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0a9104df59d04a37918377545f7fe91e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ebcff4a49ae49b989a88c7b09935a93, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/090e80391ddc4c2187aaedb5a4fe3b29] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.0 K 2024-11-18T03:17:01,167 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into a9a9f9cad1154a33be2a961b1b3e38f8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:01,167 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:01,167 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899821010; duration=0sec 2024-11-18T03:17:01,167 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:01,167 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:17:01,168 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a9104df59d04a37918377545f7fe91e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731899817678 2024-11-18T03:17:01,171 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ebcff4a49ae49b989a88c7b09935a93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731899818503 2024-11-18T03:17:01,176 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 090e80391ddc4c2187aaedb5a4fe3b29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731899819451 2024-11-18T03:17:01,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T03:17:01,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:01,208 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T03:17:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:01,236 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:01,237 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/f304f3cbe54b435585d7e6da1b1fea23 is 50, key is test_row_0/C:col10/1731899820168/Put/seqid=0 2024-11-18T03:17:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/85bd25a9c7244f12a510fc5600aa49eb is 50, key is test_row_0/A:col10/1731899820260/Put/seqid=0 2024-11-18T03:17:01,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742350_1526 (size=12663) 2024-11-18T03:17:01,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742351_1527 (size=12151) 2024-11-18T03:17:01,325 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/85bd25a9c7244f12a510fc5600aa49eb 2024-11-18T03:17:01,328 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/f304f3cbe54b435585d7e6da1b1fea23 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f304f3cbe54b435585d7e6da1b1fea23 2024-11-18T03:17:01,335 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into f304f3cbe54b435585d7e6da1b1fea23(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:01,335 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:01,335 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899821011; duration=0sec 2024-11-18T03:17:01,335 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:01,335 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:17:01,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b42514354c194d8a85c63b0dbbcb3aff is 50, key is test_row_0/B:col10/1731899820260/Put/seqid=0 2024-11-18T03:17:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T03:17:01,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742352_1528 (size=12151) 2024-11-18T03:17:01,404 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b42514354c194d8a85c63b0dbbcb3aff 2024-11-18T03:17:01,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ab37197c770742f1990b185b630c2168 is 50, key is test_row_0/C:col10/1731899820260/Put/seqid=0 2024-11-18T03:17:01,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:01,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742353_1529 (size=12151) 2024-11-18T03:17:01,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ab37197c770742f1990b185b630c2168 2024-11-18T03:17:01,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/85bd25a9c7244f12a510fc5600aa49eb as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/85bd25a9c7244f12a510fc5600aa49eb 2024-11-18T03:17:01,551 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/85bd25a9c7244f12a510fc5600aa49eb, entries=150, sequenceid=224, filesize=11.9 K 2024-11-18T03:17:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b42514354c194d8a85c63b0dbbcb3aff as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b42514354c194d8a85c63b0dbbcb3aff 2024-11-18T03:17:01,569 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b42514354c194d8a85c63b0dbbcb3aff, entries=150, sequenceid=224, filesize=11.9 K 2024-11-18T03:17:01,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ab37197c770742f1990b185b630c2168 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ab37197c770742f1990b185b630c2168 2024-11-18T03:17:01,579 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ab37197c770742f1990b185b630c2168, entries=150, sequenceid=224, filesize=11.9 K 2024-11-18T03:17:01,588 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=80.51 KB/82440 for 9d402698952bfb73679bd006311c3148 in 380ms, sequenceid=224, compaction requested=false 2024-11-18T03:17:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-18T03:17:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-18T03:17:01,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-18T03:17:01,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 545 msec 2024-11-18T03:17:01,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 558 msec 2024-11-18T03:17:01,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T03:17:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:01,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/d4f7084200d745b095b1d939a0ae85a4 is 50, key is test_row_0/A:col10/1731899821567/Put/seqid=0 2024-11-18T03:17:01,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742354_1530 (size=12151) 2024-11-18T03:17:01,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/d4f7084200d745b095b1d939a0ae85a4 2024-11-18T03:17:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T03:17:01,673 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:01,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-18T03:17:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T03:17:01,688 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:01,689 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:01,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:01,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/d9148472c89744af8f6ff3d26553bc6d is 50, key is test_row_0/B:col10/1731899821567/Put/seqid=0 2024-11-18T03:17:01,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742355_1531 (size=12151) 2024-11-18T03:17:01,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/d9148472c89744af8f6ff3d26553bc6d 2024-11-18T03:17:01,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899881684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899881685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899881689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899881690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899881694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ae39a2675c0e426ca6499447e3d9bbd1 is 50, key is test_row_0/C:col10/1731899821567/Put/seqid=0 2024-11-18T03:17:01,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742356_1532 (size=12151) 2024-11-18T03:17:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T03:17:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899881822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899881822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899881822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899881822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899881822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:01,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T03:17:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:01,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:02,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T03:17:02,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T03:17:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899882043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899882043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899882044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899882048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899882048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T03:17:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:02,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ae39a2675c0e426ca6499447e3d9bbd1 2024-11-18T03:17:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/d4f7084200d745b095b1d939a0ae85a4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d4f7084200d745b095b1d939a0ae85a4 2024-11-18T03:17:02,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d4f7084200d745b095b1d939a0ae85a4, entries=150, sequenceid=242, filesize=11.9 K 2024-11-18T03:17:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/d9148472c89744af8f6ff3d26553bc6d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d9148472c89744af8f6ff3d26553bc6d 2024-11-18T03:17:02,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d9148472c89744af8f6ff3d26553bc6d, entries=150, sequenceid=242, filesize=11.9 K 2024-11-18T03:17:02,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ae39a2675c0e426ca6499447e3d9bbd1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ae39a2675c0e426ca6499447e3d9bbd1 2024-11-18T03:17:02,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ae39a2675c0e426ca6499447e3d9bbd1, entries=150, sequenceid=242, filesize=11.9 K 2024-11-18T03:17:02,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 9d402698952bfb73679bd006311c3148 in 586ms, sequenceid=242, compaction requested=true 2024-11-18T03:17:02,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:02,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:17:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T03:17:02,189 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:02,191 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:02,191 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:17:02,191 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,191 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/a9a9f9cad1154a33be2a961b1b3e38f8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/85bd25a9c7244f12a510fc5600aa49eb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d4f7084200d745b095b1d939a0ae85a4] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.1 K 2024-11-18T03:17:02,192 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a9f9cad1154a33be2a961b1b3e38f8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731899819451 2024-11-18T03:17:02,192 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:02,192 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bd25a9c7244f12a510fc5600aa49eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731899820219 2024-11-18T03:17:02,193 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f7084200d745b095b1d939a0ae85a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731899821558 2024-11-18T03:17:02,197 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:02,197 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:17:02,197 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,197 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f304f3cbe54b435585d7e6da1b1fea23, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ab37197c770742f1990b185b630c2168, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ae39a2675c0e426ca6499447e3d9bbd1] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.1 K 2024-11-18T03:17:02,202 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f304f3cbe54b435585d7e6da1b1fea23, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731899819451 2024-11-18T03:17:02,205 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:02,205 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/e0d4ca8bb2f24ec0a4de870cd6f268dc is 50, key is test_row_0/A:col10/1731899821567/Put/seqid=0 2024-11-18T03:17:02,212 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab37197c770742f1990b185b630c2168, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731899820219 2024-11-18T03:17:02,219 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae39a2675c0e426ca6499447e3d9bbd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731899821558 2024-11-18T03:17:02,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742357_1533 (size=12765) 2024-11-18T03:17:02,262 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:02,263 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/dbe532db466a41d294b039d02eea0fed is 50, key is test_row_0/C:col10/1731899821567/Put/seqid=0 2024-11-18T03:17:02,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742358_1534 (size=12765) 2024-11-18T03:17:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T03:17:02,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T03:17:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,340 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T03:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ba64f419f79343dab2c8690a3942bdca is 50, key is test_row_0/A:col10/1731899821681/Put/seqid=0 2024-11-18T03:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:02,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:02,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742359_1535 (size=12201) 2024-11-18T03:17:02,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899882423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899882427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899882437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899882443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899882444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899882551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899882551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899882551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899882571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899882571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,706 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/e0d4ca8bb2f24ec0a4de870cd6f268dc as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e0d4ca8bb2f24ec0a4de870cd6f268dc 2024-11-18T03:17:02,718 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into e0d4ca8bb2f24ec0a4de870cd6f268dc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:02,718 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:02,718 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899822188; duration=0sec 2024-11-18T03:17:02,718 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:02,719 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:17:02,719 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:02,723 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/dbe532db466a41d294b039d02eea0fed as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/dbe532db466a41d294b039d02eea0fed 2024-11-18T03:17:02,728 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:02,729 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:17:02,729 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:02,729 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f081289136f6414a9305961438aad1a3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b42514354c194d8a85c63b0dbbcb3aff, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d9148472c89744af8f6ff3d26553bc6d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.1 K 2024-11-18T03:17:02,732 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f081289136f6414a9305961438aad1a3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731899819451 2024-11-18T03:17:02,733 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b42514354c194d8a85c63b0dbbcb3aff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731899820219 2024-11-18T03:17:02,733 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d9148472c89744af8f6ff3d26553bc6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731899821558 2024-11-18T03:17:02,743 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into dbe532db466a41d294b039d02eea0fed(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:02,743 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:02,743 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899822189; duration=0sec 2024-11-18T03:17:02,743 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:02,743 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:17:02,748 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:02,749 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/cbc690c13aec40a9b009b82d89cd1fe4 is 50, key is test_row_0/B:col10/1731899821567/Put/seqid=0 2024-11-18T03:17:02,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742360_1536 (size=12765) 2024-11-18T03:17:02,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899882771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899882772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899882772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,799 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/cbc690c13aec40a9b009b82d89cd1fe4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/cbc690c13aec40a9b009b82d89cd1fe4 2024-11-18T03:17:02,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899882791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899882791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:02,817 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into cbc690c13aec40a9b009b82d89cd1fe4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:02,817 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:02,817 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899822189; duration=0sec 2024-11-18T03:17:02,817 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:02,817 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:17:02,818 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ba64f419f79343dab2c8690a3942bdca 2024-11-18T03:17:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T03:17:02,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/245eb8a952904e98affae9472ff2662b is 50, key is test_row_0/B:col10/1731899821681/Put/seqid=0 2024-11-18T03:17:02,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742361_1537 (size=12201) 2024-11-18T03:17:03,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899883093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899883101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899883101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899883116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899883116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/245eb8a952904e98affae9472ff2662b 2024-11-18T03:17:03,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/aa467a18666340a4bb6e8068db5c5541 is 50, key is test_row_0/C:col10/1731899821681/Put/seqid=0 2024-11-18T03:17:03,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742362_1538 (size=12201) 2024-11-18T03:17:03,382 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/aa467a18666340a4bb6e8068db5c5541 2024-11-18T03:17:03,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ba64f419f79343dab2c8690a3942bdca as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ba64f419f79343dab2c8690a3942bdca 2024-11-18T03:17:03,413 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ba64f419f79343dab2c8690a3942bdca, entries=150, sequenceid=258, filesize=11.9 K 2024-11-18T03:17:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/245eb8a952904e98affae9472ff2662b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/245eb8a952904e98affae9472ff2662b 2024-11-18T03:17:03,424 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/245eb8a952904e98affae9472ff2662b, entries=150, sequenceid=258, filesize=11.9 K 2024-11-18T03:17:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/aa467a18666340a4bb6e8068db5c5541 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/aa467a18666340a4bb6e8068db5c5541 2024-11-18T03:17:03,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/aa467a18666340a4bb6e8068db5c5541, entries=150, sequenceid=258, filesize=11.9 K 2024-11-18T03:17:03,452 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9d402698952bfb73679bd006311c3148 in 1113ms, sequenceid=258, compaction requested=false 2024-11-18T03:17:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-18T03:17:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-18T03:17:03,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-18T03:17:03,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7860 sec 2024-11-18T03:17:03,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8170 sec 2024-11-18T03:17:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:03,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:17:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:03,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c13c980c386144b6a4234e07fb837635 is 50, key is test_row_0/A:col10/1731899822438/Put/seqid=0 2024-11-18T03:17:03,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742363_1539 (size=14741) 2024-11-18T03:17:03,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899883670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899883671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899883671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899883676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899883697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899883803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899883803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899883803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899883804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899883824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T03:17:03,845 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:03,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-18T03:17:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T03:17:03,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:03,863 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:03,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T03:17:04,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T03:17:04,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:04,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899884022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899884022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899884023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899884023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899884042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c13c980c386144b6a4234e07fb837635 2024-11-18T03:17:04,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/5ae45f5ec26449a4b7401026e230b95d is 50, key is test_row_0/B:col10/1731899822438/Put/seqid=0 2024-11-18T03:17:04,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742364_1540 (size=12301) 2024-11-18T03:17:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T03:17:04,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T03:17:04,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:04,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899884341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899884341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T03:17:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899884356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899884356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899884361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T03:17:04,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T03:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/5ae45f5ec26449a4b7401026e230b95d 2024-11-18T03:17:04,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/2f0939e9ff5d422a8c6e865f1342ba34 is 50, key is test_row_0/C:col10/1731899822438/Put/seqid=0 2024-11-18T03:17:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742365_1541 (size=12301) 2024-11-18T03:17:04,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/2f0939e9ff5d422a8c6e865f1342ba34 2024-11-18T03:17:04,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c13c980c386144b6a4234e07fb837635 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c13c980c386144b6a4234e07fb837635 2024-11-18T03:17:04,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T03:17:04,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:04,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:04,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c13c980c386144b6a4234e07fb837635, entries=200, sequenceid=278, filesize=14.4 K 2024-11-18T03:17:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/5ae45f5ec26449a4b7401026e230b95d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5ae45f5ec26449a4b7401026e230b95d 2024-11-18T03:17:04,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5ae45f5ec26449a4b7401026e230b95d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-18T03:17:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/2f0939e9ff5d422a8c6e865f1342ba34 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/2f0939e9ff5d422a8c6e865f1342ba34 2024-11-18T03:17:04,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/2f0939e9ff5d422a8c6e865f1342ba34, entries=150, sequenceid=278, filesize=12.0 K 2024-11-18T03:17:04,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9d402698952bfb73679bd006311c3148 in 1087ms, sequenceid=278, compaction requested=true 2024-11-18T03:17:04,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:04,710 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:04,710 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39707 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:04,710 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:17:04,710 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,711 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e0d4ca8bb2f24ec0a4de870cd6f268dc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ba64f419f79343dab2c8690a3942bdca, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c13c980c386144b6a4234e07fb837635] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=38.8 K 2024-11-18T03:17:04,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:04,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:04,711 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d4ca8bb2f24ec0a4de870cd6f268dc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731899821558 2024-11-18T03:17:04,711 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:04,711 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba64f419f79343dab2c8690a3942bdca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731899821675 2024-11-18T03:17:04,712 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c13c980c386144b6a4234e07fb837635, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731899822418 2024-11-18T03:17:04,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:04,712 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:04,713 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:17:04,713 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,713 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/cbc690c13aec40a9b009b82d89cd1fe4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/245eb8a952904e98affae9472ff2662b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5ae45f5ec26449a4b7401026e230b95d] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.4 K 2024-11-18T03:17:04,713 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc690c13aec40a9b009b82d89cd1fe4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731899821558 2024-11-18T03:17:04,713 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 245eb8a952904e98affae9472ff2662b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731899821675 2024-11-18T03:17:04,714 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae45f5ec26449a4b7401026e230b95d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731899822418 2024-11-18T03:17:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:04,742 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:04,743 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/7c7481370f2449808586175b63a51ad9 is 50, key is test_row_0/A:col10/1731899822438/Put/seqid=0 2024-11-18T03:17:04,758 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:04,759 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/81cd49f026df4ff0836373f0768e650d is 50, key is test_row_0/B:col10/1731899822438/Put/seqid=0 2024-11-18T03:17:04,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742366_1542 (size=13017) 2024-11-18T03:17:04,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742367_1543 (size=13017) 2024-11-18T03:17:04,831 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/7c7481370f2449808586175b63a51ad9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7c7481370f2449808586175b63a51ad9 2024-11-18T03:17:04,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T03:17:04,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:17:04,847 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into 7c7481370f2449808586175b63a51ad9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:04,847 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:04,847 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899824709; duration=0sec 2024-11-18T03:17:04,847 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:04,847 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:17:04,847 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:04,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:04,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:04,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:04,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:04,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:04,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:04,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:04,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:17:04,849 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:04,849 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/dbe532db466a41d294b039d02eea0fed, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/aa467a18666340a4bb6e8068db5c5541, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/2f0939e9ff5d422a8c6e865f1342ba34] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.4 K 2024-11-18T03:17:04,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe532db466a41d294b039d02eea0fed, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731899821558 2024-11-18T03:17:04,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa467a18666340a4bb6e8068db5c5541, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731899821675 2024-11-18T03:17:04,850 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f0939e9ff5d422a8c6e865f1342ba34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731899822418 2024-11-18T03:17:04,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/edc04a647a1842458504978c1beee867 is 50, key is test_row_0/A:col10/1731899823668/Put/seqid=0 2024-11-18T03:17:04,870 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:04,871 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/983425afbfc64dd18ea140daf6bc4b42 is 50, key is test_row_0/C:col10/1731899822438/Put/seqid=0 2024-11-18T03:17:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:04,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:04,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742368_1544 (size=12301) 2024-11-18T03:17:04,917 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/edc04a647a1842458504978c1beee867 2024-11-18T03:17:04,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742369_1545 (size=13017) 2024-11-18T03:17:04,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/1f83ecbbaabc4ef6aa4c69eac6bc329e is 50, key is test_row_0/B:col10/1731899823668/Put/seqid=0 2024-11-18T03:17:04,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899884958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742370_1546 (size=12301) 2024-11-18T03:17:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,986 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/1f83ecbbaabc4ef6aa4c69eac6bc329e 2024-11-18T03:17:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899884963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899884963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899884966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899884965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T03:17:05,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9ec2c847fbed48b7b4584f4e1ff46ec3 is 50, key is test_row_0/C:col10/1731899823668/Put/seqid=0 2024-11-18T03:17:05,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742371_1547 (size=12301) 2024-11-18T03:17:05,051 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9ec2c847fbed48b7b4584f4e1ff46ec3 2024-11-18T03:17:05,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/edc04a647a1842458504978c1beee867 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/edc04a647a1842458504978c1beee867 2024-11-18T03:17:05,059 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/edc04a647a1842458504978c1beee867, entries=150, sequenceid=294, filesize=12.0 K 2024-11-18T03:17:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/1f83ecbbaabc4ef6aa4c69eac6bc329e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/1f83ecbbaabc4ef6aa4c69eac6bc329e 2024-11-18T03:17:05,064 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/1f83ecbbaabc4ef6aa4c69eac6bc329e, entries=150, sequenceid=294, filesize=12.0 K 2024-11-18T03:17:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/9ec2c847fbed48b7b4584f4e1ff46ec3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ec2c847fbed48b7b4584f4e1ff46ec3 2024-11-18T03:17:05,070 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ec2c847fbed48b7b4584f4e1ff46ec3, entries=150, sequenceid=294, filesize=12.0 K 2024-11-18T03:17:05,071 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9d402698952bfb73679bd006311c3148 in 226ms, sequenceid=294, compaction requested=false 2024-11-18T03:17:05,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:05,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:05,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-18T03:17:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-18T03:17:05,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-18T03:17:05,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2120 sec 2024-11-18T03:17:05,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.2290 sec 2024-11-18T03:17:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:05,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:17:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:05,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/096bd4a8fe2f42afb5c50ccbb3a7ed27 is 50, key is test_row_0/A:col10/1731899825104/Put/seqid=0 2024-11-18T03:17:05,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742372_1548 (size=12301) 2024-11-18T03:17:05,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/096bd4a8fe2f42afb5c50ccbb3a7ed27 2024-11-18T03:17:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899885149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899885151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b49813c7d7a64cd391e465d69f18fc2a is 50, key is test_row_0/B:col10/1731899825104/Put/seqid=0 2024-11-18T03:17:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899885157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899885160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899885164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742373_1549 (size=12301) 2024-11-18T03:17:05,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b49813c7d7a64cd391e465d69f18fc2a 2024-11-18T03:17:05,247 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/81cd49f026df4ff0836373f0768e650d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/81cd49f026df4ff0836373f0768e650d 2024-11-18T03:17:05,258 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into 81cd49f026df4ff0836373f0768e650d(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:05,258 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:05,258 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899824711; duration=0sec 2024-11-18T03:17:05,258 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:05,258 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:17:05,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/f0905435da8b454ab6e5550200156c95 is 50, key is test_row_0/C:col10/1731899825104/Put/seqid=0 2024-11-18T03:17:05,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899885271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899885285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899885291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742374_1550 (size=12301) 2024-11-18T03:17:05,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/f0905435da8b454ab6e5550200156c95 2024-11-18T03:17:05,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899885305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899885306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,353 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/983425afbfc64dd18ea140daf6bc4b42 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/983425afbfc64dd18ea140daf6bc4b42 2024-11-18T03:17:05,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/096bd4a8fe2f42afb5c50ccbb3a7ed27 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/096bd4a8fe2f42afb5c50ccbb3a7ed27 2024-11-18T03:17:05,369 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into 983425afbfc64dd18ea140daf6bc4b42(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:05,369 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:05,369 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899824721; duration=0sec 2024-11-18T03:17:05,369 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:05,370 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:17:05,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/096bd4a8fe2f42afb5c50ccbb3a7ed27, entries=150, sequenceid=313, filesize=12.0 K 2024-11-18T03:17:05,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b49813c7d7a64cd391e465d69f18fc2a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b49813c7d7a64cd391e465d69f18fc2a 2024-11-18T03:17:05,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b49813c7d7a64cd391e465d69f18fc2a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-18T03:17:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/f0905435da8b454ab6e5550200156c95 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f0905435da8b454ab6e5550200156c95 2024-11-18T03:17:05,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f0905435da8b454ab6e5550200156c95, entries=150, sequenceid=313, filesize=12.0 K 2024-11-18T03:17:05,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9d402698952bfb73679bd006311c3148 in 317ms, sequenceid=313, compaction requested=true 2024-11-18T03:17:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:05,425 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:17:05,430 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:05,430 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:17:05,430 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:05,430 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7c7481370f2449808586175b63a51ad9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/edc04a647a1842458504978c1beee867, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/096bd4a8fe2f42afb5c50ccbb3a7ed27] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.7 K 2024-11-18T03:17:05,430 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7481370f2449808586175b63a51ad9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731899822418 2024-11-18T03:17:05,431 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting edc04a647a1842458504978c1beee867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731899823642 2024-11-18T03:17:05,431 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 096bd4a8fe2f42afb5c50ccbb3a7ed27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731899824957 2024-11-18T03:17:05,432 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:05,452 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:05,452 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:17:05,452 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:05,452 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/81cd49f026df4ff0836373f0768e650d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/1f83ecbbaabc4ef6aa4c69eac6bc329e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b49813c7d7a64cd391e465d69f18fc2a] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.7 K 2024-11-18T03:17:05,453 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 81cd49f026df4ff0836373f0768e650d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731899822418 2024-11-18T03:17:05,453 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f83ecbbaabc4ef6aa4c69eac6bc329e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731899823642 2024-11-18T03:17:05,456 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting b49813c7d7a64cd391e465d69f18fc2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731899824957 2024-11-18T03:17:05,478 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:05,479 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ca089b29e4a744019f2cbcc6ce88d744 is 50, key is test_row_0/A:col10/1731899825104/Put/seqid=0 2024-11-18T03:17:05,506 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:05,507 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/3a86e916d2a54b848d68344aff49e2a2 is 50, key is test_row_0/B:col10/1731899825104/Put/seqid=0 2024-11-18T03:17:05,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742375_1551 (size=13119) 2024-11-18T03:17:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:05,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T03:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:05,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742376_1552 (size=13119) 2024-11-18T03:17:05,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/f6669744a764491b9b5c9affcc7cd461 is 50, key is test_row_0/A:col10/1731899825125/Put/seqid=0 2024-11-18T03:17:05,577 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/ca089b29e4a744019f2cbcc6ce88d744 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ca089b29e4a744019f2cbcc6ce88d744 2024-11-18T03:17:05,593 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/3a86e916d2a54b848d68344aff49e2a2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/3a86e916d2a54b848d68344aff49e2a2 2024-11-18T03:17:05,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742377_1553 (size=14741) 2024-11-18T03:17:05,614 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into ca089b29e4a744019f2cbcc6ce88d744(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:05,614 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:05,614 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899825425; duration=0sec 2024-11-18T03:17:05,614 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:05,614 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:17:05,614 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:05,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899885588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899885591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,620 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:05,620 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:17:05,620 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:05,620 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/983425afbfc64dd18ea140daf6bc4b42, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ec2c847fbed48b7b4584f4e1ff46ec3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f0905435da8b454ab6e5550200156c95] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=36.7 K 2024-11-18T03:17:05,628 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 983425afbfc64dd18ea140daf6bc4b42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731899822418 2024-11-18T03:17:05,632 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into 3a86e916d2a54b848d68344aff49e2a2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:05,632 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:05,633 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899825425; duration=0sec 2024-11-18T03:17:05,633 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:05,633 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:17:05,634 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec2c847fbed48b7b4584f4e1ff46ec3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731899823642 2024-11-18T03:17:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899885598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,640 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0905435da8b454ab6e5550200156c95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731899824957 2024-11-18T03:17:05,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899885611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899885602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899885721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899885722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899885741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899885755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,773 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#471 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:05,773 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/c11be004a0bf43d08bf18bb2f54a2192 is 50, key is test_row_0/C:col10/1731899825104/Put/seqid=0 2024-11-18T03:17:05,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899885763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742378_1554 (size=13119) 2024-11-18T03:17:05,880 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/c11be004a0bf43d08bf18bb2f54a2192 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c11be004a0bf43d08bf18bb2f54a2192 2024-11-18T03:17:05,896 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into c11be004a0bf43d08bf18bb2f54a2192(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:05,896 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:05,896 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899825425; duration=0sec 2024-11-18T03:17:05,897 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:05,897 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:17:05,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899885943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899885943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899885961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899885972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:05,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/f6669744a764491b9b5c9affcc7cd461 2024-11-18T03:17:06,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899885992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T03:17:06,025 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:06,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-18T03:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T03:17:06,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:06,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:06,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:06,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b6516ea439a448248afa629f637b7f6e is 50, key is test_row_0/B:col10/1731899825125/Put/seqid=0 2024-11-18T03:17:06,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742379_1555 (size=12301) 2024-11-18T03:17:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T03:17:06,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T03:17:06,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:06,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899886264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899886265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899886285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899886302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899886311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T03:17:06,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T03:17:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T03:17:06,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b6516ea439a448248afa629f637b7f6e 2024-11-18T03:17:06,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/adce04966d1944d78f0e3730965134f2 is 50, key is test_row_0/C:col10/1731899825125/Put/seqid=0 2024-11-18T03:17:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T03:17:06,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T03:17:06,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:06,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742380_1556 (size=12301) 2024-11-18T03:17:06,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/adce04966d1944d78f0e3730965134f2 2024-11-18T03:17:06,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/f6669744a764491b9b5c9affcc7cd461 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/f6669744a764491b9b5c9affcc7cd461 2024-11-18T03:17:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899886782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899886782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/f6669744a764491b9b5c9affcc7cd461, entries=200, sequenceid=329, filesize=14.4 K 2024-11-18T03:17:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/b6516ea439a448248afa629f637b7f6e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6516ea439a448248afa629f637b7f6e 2024-11-18T03:17:06,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899886811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899886824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T03:17:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:06,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:06,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6516ea439a448248afa629f637b7f6e, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T03:17:06,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/adce04966d1944d78f0e3730965134f2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/adce04966d1944d78f0e3730965134f2 2024-11-18T03:17:06,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899886841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:06,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/adce04966d1944d78f0e3730965134f2, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T03:17:06,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9d402698952bfb73679bd006311c3148 in 1345ms, sequenceid=329, compaction requested=false 2024-11-18T03:17:06,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:06,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T03:17:07,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:07,000 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:17:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/cf478e97dba64c48ab594856d77c1cee is 50, key is test_row_0/A:col10/1731899825602/Put/seqid=0 2024-11-18T03:17:07,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742381_1557 (size=9857) 2024-11-18T03:17:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T03:17:07,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/cf478e97dba64c48ab594856d77c1cee 2024-11-18T03:17:07,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/de72de6806ef4829a3e0f9bc0773c545 is 50, key is test_row_0/B:col10/1731899825602/Put/seqid=0 2024-11-18T03:17:07,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742382_1558 (size=9857) 2024-11-18T03:17:07,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:07,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899887858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899887861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899887864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899887864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899887876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,888 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 9d402698952bfb73679bd006311c3148 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:17:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,914 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/de72de6806ef4829a3e0f9bc0773c545 2024-11-18T03:17:07,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/c0c291619d28442b8da65918b68ac57e is 50, key is test_row_0/C:col10/1731899825602/Put/seqid=0 2024-11-18T03:17:07,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899887981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899887981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899887981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:07,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742383_1559 (size=9857) 2024-11-18T03:17:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899887996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T03:17:08,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899888200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899888200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899888201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899888229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,396 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/c0c291619d28442b8da65918b68ac57e 2024-11-18T03:17:08,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/cf478e97dba64c48ab594856d77c1cee as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/cf478e97dba64c48ab594856d77c1cee 2024-11-18T03:17:08,492 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/cf478e97dba64c48ab594856d77c1cee, entries=100, sequenceid=350, filesize=9.6 K 2024-11-18T03:17:08,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/de72de6806ef4829a3e0f9bc0773c545 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/de72de6806ef4829a3e0f9bc0773c545 2024-11-18T03:17:08,506 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/de72de6806ef4829a3e0f9bc0773c545, entries=100, sequenceid=350, filesize=9.6 K 2024-11-18T03:17:08,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/c0c291619d28442b8da65918b68ac57e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c0c291619d28442b8da65918b68ac57e 2024-11-18T03:17:08,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c0c291619d28442b8da65918b68ac57e, entries=100, sequenceid=350, filesize=9.6 K 2024-11-18T03:17:08,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9d402698952bfb73679bd006311c3148 in 1522ms, sequenceid=350, compaction requested=true 2024-11-18T03:17:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-18T03:17:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-18T03:17:08,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-18T03:17:08,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4950 sec 2024-11-18T03:17:08,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.5030 sec 2024-11-18T03:17:08,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T03:17:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/e09aa98e48f445f389250247d30a2d42 is 50, key is test_row_0/A:col10/1731899827859/Put/seqid=0 2024-11-18T03:17:08,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742384_1560 (size=12301) 2024-11-18T03:17:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899888609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899888615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899888622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899888633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899888742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899888742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899888751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899888763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899888963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899888963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899888974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899888984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/e09aa98e48f445f389250247d30a2d42 2024-11-18T03:17:09,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/94be603224374d22bdf4a88a67d50706 is 50, key is test_row_0/B:col10/1731899827859/Put/seqid=0 2024-11-18T03:17:09,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742385_1561 (size=12301) 2024-11-18T03:17:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899889282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899889282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899889282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899889311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/94be603224374d22bdf4a88a67d50706 2024-11-18T03:17:09,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/65a4485459f7437180a092f638f0a3e3 is 50, key is test_row_0/C:col10/1731899827859/Put/seqid=0 2024-11-18T03:17:09,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742386_1562 (size=12301) 2024-11-18T03:17:09,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899889804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899889804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899889804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899889833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33908 deadline: 1731899889911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:09,924 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_1' in 9d402698952bfb73679bd006311c3148 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4322 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:17:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:09,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/65a4485459f7437180a092f638f0a3e3 2024-11-18T03:17:09,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/e09aa98e48f445f389250247d30a2d42 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e09aa98e48f445f389250247d30a2d42 2024-11-18T03:17:09,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e09aa98e48f445f389250247d30a2d42, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T03:17:09,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/94be603224374d22bdf4a88a67d50706 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/94be603224374d22bdf4a88a67d50706 2024-11-18T03:17:09,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/94be603224374d22bdf4a88a67d50706, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T03:17:09,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/65a4485459f7437180a092f638f0a3e3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/65a4485459f7437180a092f638f0a3e3 2024-11-18T03:17:09,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/65a4485459f7437180a092f638f0a3e3, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T03:17:09,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9d402698952bfb73679bd006311c3148 in 1460ms, sequenceid=365, compaction requested=true 2024-11-18T03:17:09,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:09,997 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:17:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:17:09,998 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:17:09,999 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:17:09,999 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:09,999 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ca089b29e4a744019f2cbcc6ce88d744, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/f6669744a764491b9b5c9affcc7cd461, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/cf478e97dba64c48ab594856d77c1cee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e09aa98e48f445f389250247d30a2d42] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=48.8 K 2024-11-18T03:17:09,999 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ca089b29e4a744019f2cbcc6ce88d744, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731899824957 2024-11-18T03:17:09,999 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f6669744a764491b9b5c9affcc7cd461, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731899825125 2024-11-18T03:17:09,999 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting cf478e97dba64c48ab594856d77c1cee, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731899825599 2024-11-18T03:17:10,000 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e09aa98e48f445f389250247d30a2d42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899827859 2024-11-18T03:17:10,000 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:17:10,002 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:17:10,002 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:17:10,002 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:10,003 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/3a86e916d2a54b848d68344aff49e2a2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6516ea439a448248afa629f637b7f6e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/de72de6806ef4829a3e0f9bc0773c545, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/94be603224374d22bdf4a88a67d50706] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=46.5 K 2024-11-18T03:17:10,003 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a86e916d2a54b848d68344aff49e2a2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731899824957 2024-11-18T03:17:10,003 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6516ea439a448248afa629f637b7f6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731899825125 2024-11-18T03:17:10,004 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting de72de6806ef4829a3e0f9bc0773c545, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731899825599 2024-11-18T03:17:10,004 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94be603224374d22bdf4a88a67d50706, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899827859 2024-11-18T03:17:10,010 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:10,010 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/5d13da8b0e08478fb4a479e6fc8e49b7 is 50, key is test_row_0/A:col10/1731899827859/Put/seqid=0 2024-11-18T03:17:10,029 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:10,029 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/fdd15fb5a72d47d6b9b99bd014c83e04 is 50, key is test_row_0/B:col10/1731899827859/Put/seqid=0 2024-11-18T03:17:10,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742387_1563 (size=13255) 2024-11-18T03:17:10,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742388_1564 (size=13255) 2024-11-18T03:17:10,129 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/fdd15fb5a72d47d6b9b99bd014c83e04 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fdd15fb5a72d47d6b9b99bd014c83e04 2024-11-18T03:17:10,162 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into fdd15fb5a72d47d6b9b99bd014c83e04(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:10,162 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:10,162 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=12, startTime=1731899829998; duration=0sec 2024-11-18T03:17:10,163 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:10,163 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:17:10,163 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T03:17:10,180 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T03:17:10,180 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:17:10,180 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:10,180 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c11be004a0bf43d08bf18bb2f54a2192, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/adce04966d1944d78f0e3730965134f2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c0c291619d28442b8da65918b68ac57e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/65a4485459f7437180a092f638f0a3e3] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=46.5 K 2024-11-18T03:17:10,183 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c11be004a0bf43d08bf18bb2f54a2192, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731899824957 2024-11-18T03:17:10,185 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting adce04966d1944d78f0e3730965134f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731899825125 2024-11-18T03:17:10,188 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c291619d28442b8da65918b68ac57e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731899825599 2024-11-18T03:17:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T03:17:10,193 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:10,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:10,197 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65a4485459f7437180a092f638f0a3e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899827859 2024-11-18T03:17:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-18T03:17:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T03:17:10,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:10,204 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:10,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:10,271 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:10,271 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/305ab37facab41eca2c67f1620c76bd9 is 50, key is test_row_0/C:col10/1731899827859/Put/seqid=0 2024-11-18T03:17:10,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742389_1565 (size=13255) 2024-11-18T03:17:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T03:17:10,339 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/305ab37facab41eca2c67f1620c76bd9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/305ab37facab41eca2c67f1620c76bd9 2024-11-18T03:17:10,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T03:17:10,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:10,358 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T03:17:10,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:10,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:10,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:10,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:10,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:10,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:10,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/6762a651870f4e6b80c87678b72f552e is 50, key is test_row_0/A:col10/1731899828621/Put/seqid=0 2024-11-18T03:17:10,372 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into 305ab37facab41eca2c67f1620c76bd9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:10,372 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:10,372 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=12, startTime=1731899829998; duration=0sec 2024-11-18T03:17:10,372 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:10,372 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:17:10,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742390_1566 (size=12301) 2024-11-18T03:17:10,411 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/6762a651870f4e6b80c87678b72f552e 2024-11-18T03:17:10,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/23c423d69d924b36ac64422bd642ebd9 is 50, key is test_row_0/B:col10/1731899828621/Put/seqid=0 2024-11-18T03:17:10,477 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/5d13da8b0e08478fb4a479e6fc8e49b7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/5d13da8b0e08478fb4a479e6fc8e49b7 2024-11-18T03:17:10,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742391_1567 (size=12301) 2024-11-18T03:17:10,491 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into 5d13da8b0e08478fb4a479e6fc8e49b7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:10,491 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:10,491 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=12, startTime=1731899829997; duration=0sec 2024-11-18T03:17:10,491 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:10,491 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:17:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T03:17:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T03:17:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. as already flushing 2024-11-18T03:17:10,895 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/23c423d69d924b36ac64422bd642ebd9 2024-11-18T03:17:10,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ff892103e1944c01852aefa37534bca7 is 50, key is test_row_0/C:col10/1731899828621/Put/seqid=0 2024-11-18T03:17:10,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742392_1568 (size=12301) 2024-11-18T03:17:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899890937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899890937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:10,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899890940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:10,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899890968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899891071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899891071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899891071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899891092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,223 INFO [Thread-2144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2144. 2024-11-18T03:17:11,223 INFO [Thread-2146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2146. 2024-11-18T03:17:11,223 DEBUG [Thread-2144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,223 DEBUG [Thread-2146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,223 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,223 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,223 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,223 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,227 INFO [Thread-2142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2142. 2024-11-18T03:17:11,227 DEBUG [Thread-2142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,227 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,228 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,228 INFO [Thread-2150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2150. 2024-11-18T03:17:11,228 DEBUG [Thread-2150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,228 DEBUG [Thread-2150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,228 DEBUG [Thread-2150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,235 INFO [Thread-2148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2148. 2024-11-18T03:17:11,235 DEBUG [Thread-2148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,235 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,235 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33922 deadline: 1731899891281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731899891282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33898 deadline: 1731899891282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33894 deadline: 1731899891311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 , the old value is region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., hostname=fd862c3a04f9,38549,1731899684479, seqNum=2 because the exception is null or not the one we care about 2024-11-18T03:17:11,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ff892103e1944c01852aefa37534bca7 2024-11-18T03:17:11,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/6762a651870f4e6b80c87678b72f552e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6762a651870f4e6b80c87678b72f552e 2024-11-18T03:17:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T03:17:11,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6762a651870f4e6b80c87678b72f552e, entries=150, sequenceid=383, filesize=12.0 K 2024-11-18T03:17:11,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/23c423d69d924b36ac64422bd642ebd9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/23c423d69d924b36ac64422bd642ebd9 2024-11-18T03:17:11,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/23c423d69d924b36ac64422bd642ebd9, entries=150, sequenceid=383, filesize=12.0 K 2024-11-18T03:17:11,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/ff892103e1944c01852aefa37534bca7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ff892103e1944c01852aefa37534bca7 2024-11-18T03:17:11,350 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ff892103e1944c01852aefa37534bca7, entries=150, sequenceid=383, filesize=12.0 K 2024-11-18T03:17:11,351 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9d402698952bfb73679bd006311c3148 in 992ms, sequenceid=383, compaction requested=false 2024-11-18T03:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-18T03:17:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-18T03:17:11,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-18T03:17:11,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-11-18T03:17:11,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.1600 sec 2024-11-18T03:17:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:11,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:17:11,593 INFO [Thread-2139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2139. 2024-11-18T03:17:11,593 INFO [Thread-2137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2137. 2024-11-18T03:17:11,593 INFO [Thread-2133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2133. 2024-11-18T03:17:11,593 DEBUG [Thread-2139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,593 DEBUG [Thread-2133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,593 DEBUG [Thread-2137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,593 DEBUG [Thread-2139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,593 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,593 DEBUG [Thread-2137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,594 DEBUG [Thread-2137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,594 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,594 DEBUG [Thread-2139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:11,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:11,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:11,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c703621071254c949c9a277425eec659 is 50, key is test_row_0/A:col10/1731899831591/Put/seqid=0 2024-11-18T03:17:11,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742393_1569 (size=12301) 2024-11-18T03:17:11,623 INFO [Thread-2135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2135. 2024-11-18T03:17:11,623 DEBUG [Thread-2135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:11,623 DEBUG [Thread-2135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,623 DEBUG [Thread-2135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:11,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:12,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c703621071254c949c9a277425eec659 2024-11-18T03:17:12,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/65e8e111f6a247ffbc2beedf33264ab0 is 50, key is test_row_0/B:col10/1731899831591/Put/seqid=0 2024-11-18T03:17:12,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742394_1570 (size=12301) 2024-11-18T03:17:12,217 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T03:17:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T03:17:12,351 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:12,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/65e8e111f6a247ffbc2beedf33264ab0 2024-11-18T03:17:12,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/a1affa1c41e243a889d186ffd0b46d0a is 50, key is test_row_0/C:col10/1731899831591/Put/seqid=0 2024-11-18T03:17:12,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742395_1571 (size=12301) 2024-11-18T03:17:12,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/a1affa1c41e243a889d186ffd0b46d0a 2024-11-18T03:17:12,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/c703621071254c949c9a277425eec659 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c703621071254c949c9a277425eec659 2024-11-18T03:17:12,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c703621071254c949c9a277425eec659, entries=150, sequenceid=403, filesize=12.0 K 2024-11-18T03:17:12,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/65e8e111f6a247ffbc2beedf33264ab0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/65e8e111f6a247ffbc2beedf33264ab0 2024-11-18T03:17:12,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/65e8e111f6a247ffbc2beedf33264ab0, entries=150, sequenceid=403, filesize=12.0 K 2024-11-18T03:17:12,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/a1affa1c41e243a889d186ffd0b46d0a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/a1affa1c41e243a889d186ffd0b46d0a 2024-11-18T03:17:12,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/a1affa1c41e243a889d186ffd0b46d0a, entries=150, sequenceid=403, filesize=12.0 K 2024-11-18T03:17:12,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for 9d402698952bfb73679bd006311c3148 in 1249ms, sequenceid=403, compaction requested=true 2024-11-18T03:17:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:12,842 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d402698952bfb73679bd006311c3148:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:12,842 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/A is initiating minor compaction (all files) 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/B is initiating minor compaction (all files) 2024-11-18T03:17:12,843 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/A in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:12,843 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/B in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:12,843 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/5d13da8b0e08478fb4a479e6fc8e49b7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6762a651870f4e6b80c87678b72f552e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c703621071254c949c9a277425eec659] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=37.0 K 2024-11-18T03:17:12,843 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fdd15fb5a72d47d6b9b99bd014c83e04, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/23c423d69d924b36ac64422bd642ebd9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/65e8e111f6a247ffbc2beedf33264ab0] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=37.0 K 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d13da8b0e08478fb4a479e6fc8e49b7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899827859 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdd15fb5a72d47d6b9b99bd014c83e04, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899827859 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6762a651870f4e6b80c87678b72f552e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731899828604 2024-11-18T03:17:12,843 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c423d69d924b36ac64422bd642ebd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731899828604 2024-11-18T03:17:12,844 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c703621071254c949c9a277425eec659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731899831591 2024-11-18T03:17:12,844 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e8e111f6a247ffbc2beedf33264ab0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731899831591 2024-11-18T03:17:12,849 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#A#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:12,849 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#B#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:12,849 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/54560340eeca4dcb936f5e046aa124ca is 50, key is test_row_0/A:col10/1731899831591/Put/seqid=0 2024-11-18T03:17:12,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/d2f430a6bf374ab0a2e04dfd5845639e is 50, key is test_row_0/B:col10/1731899831591/Put/seqid=0 2024-11-18T03:17:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742396_1572 (size=13357) 2024-11-18T03:17:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742397_1573 (size=13357) 2024-11-18T03:17:13,255 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/54560340eeca4dcb936f5e046aa124ca as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/54560340eeca4dcb936f5e046aa124ca 2024-11-18T03:17:13,255 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/d2f430a6bf374ab0a2e04dfd5845639e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d2f430a6bf374ab0a2e04dfd5845639e 2024-11-18T03:17:13,258 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/B of 9d402698952bfb73679bd006311c3148 into d2f430a6bf374ab0a2e04dfd5845639e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:13,258 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/A of 9d402698952bfb73679bd006311c3148 into 54560340eeca4dcb936f5e046aa124ca(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:13,258 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:13,258 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:13,258 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/B, priority=13, startTime=1731899832842; duration=0sec 2024-11-18T03:17:13,258 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/A, priority=13, startTime=1731899832842; duration=0sec 2024-11-18T03:17:13,258 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:13,258 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:13,258 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:B 2024-11-18T03:17:13,258 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:A 2024-11-18T03:17:13,258 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:13,259 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:13,259 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 9d402698952bfb73679bd006311c3148/C is initiating minor compaction (all files) 2024-11-18T03:17:13,259 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d402698952bfb73679bd006311c3148/C in TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:13,259 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/305ab37facab41eca2c67f1620c76bd9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ff892103e1944c01852aefa37534bca7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/a1affa1c41e243a889d186ffd0b46d0a] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp, totalSize=37.0 K 2024-11-18T03:17:13,259 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 305ab37facab41eca2c67f1620c76bd9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731899827859 2024-11-18T03:17:13,259 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff892103e1944c01852aefa37534bca7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731899828604 2024-11-18T03:17:13,260 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1affa1c41e243a889d186ffd0b46d0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731899831591 2024-11-18T03:17:13,267 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d402698952bfb73679bd006311c3148#C#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:13,267 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/b5630a12d1c84c16975a07819816b0ea is 50, key is test_row_0/C:col10/1731899831591/Put/seqid=0 2024-11-18T03:17:13,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742398_1574 (size=13357) 2024-11-18T03:17:13,675 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/b5630a12d1c84c16975a07819816b0ea as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/b5630a12d1c84c16975a07819816b0ea 2024-11-18T03:17:13,680 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d402698952bfb73679bd006311c3148/C of 9d402698952bfb73679bd006311c3148 into b5630a12d1c84c16975a07819816b0ea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:13,680 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d402698952bfb73679bd006311c3148: 2024-11-18T03:17:13,680 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., storeName=9d402698952bfb73679bd006311c3148/C, priority=13, startTime=1731899832842; duration=0sec 2024-11-18T03:17:13,680 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:13,680 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d402698952bfb73679bd006311c3148:C 2024-11-18T03:17:13,952 INFO [Thread-2131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2131. 2024-11-18T03:17:13,952 DEBUG [Thread-2131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:13,952 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:13,953 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:13,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 590 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1762 rows 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 577 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1731 rows 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 581 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1734 rows 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 572 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1714 rows 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 571 2024-11-18T03:17:13,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1713 rows 2024-11-18T03:17:13,953 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:17:13,953 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:17:13,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:13,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:13,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:13,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-18T03:17:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T03:17:13,961 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899833960"}]},"ts":"1731899833960"} 2024-11-18T03:17:13,963 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T03:17:13,963 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T03:17:13,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:17:13,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, UNASSIGN}] 2024-11-18T03:17:13,967 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, UNASSIGN 2024-11-18T03:17:13,968 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=9d402698952bfb73679bd006311c3148, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:13,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, UNASSIGN because future has completed 2024-11-18T03:17:13,970 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:17:13,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:17:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T03:17:14,124 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:14,124 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:17:14,124 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 9d402698952bfb73679bd006311c3148, disabling compactions & flushes 2024-11-18T03:17:14,124 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:14,124 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:14,124 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. after waiting 0 ms 2024-11-18T03:17:14,124 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:14,124 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 9d402698952bfb73679bd006311c3148 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-18T03:17:14,125 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=A 2024-11-18T03:17:14,125 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:14,125 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=B 2024-11-18T03:17:14,125 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:14,125 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d402698952bfb73679bd006311c3148, store=C 2024-11-18T03:17:14,125 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:14,129 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/26d5f64e6ce048c88f036da4db3bcba7 is 50, key is test_row_1/A:col10/1731899833951/Put/seqid=0 2024-11-18T03:17:14,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742399_1575 (size=7415) 2024-11-18T03:17:14,149 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/26d5f64e6ce048c88f036da4db3bcba7 2024-11-18T03:17:14,157 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/9eca1519f819499298c0e46137e8536e is 50, key is test_row_1/B:col10/1731899833951/Put/seqid=0 2024-11-18T03:17:14,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742400_1576 (size=7415) 2024-11-18T03:17:14,166 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/9eca1519f819499298c0e46137e8536e 2024-11-18T03:17:14,174 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/0d81a4a90c22402ea3d150b9fd047872 is 50, key is test_row_1/C:col10/1731899833951/Put/seqid=0 2024-11-18T03:17:14,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742401_1577 (size=7415) 2024-11-18T03:17:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T03:17:14,579 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/0d81a4a90c22402ea3d150b9fd047872 2024-11-18T03:17:14,584 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/A/26d5f64e6ce048c88f036da4db3bcba7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/26d5f64e6ce048c88f036da4db3bcba7 2024-11-18T03:17:14,588 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/26d5f64e6ce048c88f036da4db3bcba7, entries=50, sequenceid=411, filesize=7.2 K 2024-11-18T03:17:14,589 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/B/9eca1519f819499298c0e46137e8536e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9eca1519f819499298c0e46137e8536e 2024-11-18T03:17:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T03:17:14,593 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9eca1519f819499298c0e46137e8536e, entries=50, sequenceid=411, filesize=7.2 K 2024-11-18T03:17:14,594 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/.tmp/C/0d81a4a90c22402ea3d150b9fd047872 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0d81a4a90c22402ea3d150b9fd047872 2024-11-18T03:17:14,598 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0d81a4a90c22402ea3d150b9fd047872, entries=50, sequenceid=411, filesize=7.2 K 2024-11-18T03:17:14,598 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 9d402698952bfb73679bd006311c3148 in 474ms, sequenceid=411, compaction requested=false 2024-11-18T03:17:14,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d15272a30aeb427fb9a3a67306c2d622, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/35de357400244c6797afed84dcd023f3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ad7fecc4755e4c0f8f4cddf40cfc67a3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e078efde167b44218ed1ce66e7ffb99a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/8f4982741f8d43edb380d1afa1351217, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2358409e02454d7ab000452fe12c0f7c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/42b12ef709334691be68848204b73743, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7220092a4f584bec81120ef0cf5b80ea, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c1de0445bb6d4529bb301529164a7346, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2c9acecf5d4d44f7b56edd886f75dc48, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/92ea590ddfa44c8f9f97769bf4179a08, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/84fe4196fcaa42f0a8f102596955cdb4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6f60670d40bf4ae9ae2d342e5ea96bfe, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/9db474b8a2154c828888133124b05ade, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/4f29622273dc4747a1b420f7217387b7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/a9a9f9cad1154a33be2a961b1b3e38f8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/85bd25a9c7244f12a510fc5600aa49eb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e0d4ca8bb2f24ec0a4de870cd6f268dc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d4f7084200d745b095b1d939a0ae85a4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ba64f419f79343dab2c8690a3942bdca, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c13c980c386144b6a4234e07fb837635, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7c7481370f2449808586175b63a51ad9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/edc04a647a1842458504978c1beee867, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ca089b29e4a744019f2cbcc6ce88d744, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/096bd4a8fe2f42afb5c50ccbb3a7ed27, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/f6669744a764491b9b5c9affcc7cd461, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/cf478e97dba64c48ab594856d77c1cee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/5d13da8b0e08478fb4a479e6fc8e49b7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e09aa98e48f445f389250247d30a2d42, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6762a651870f4e6b80c87678b72f552e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c703621071254c949c9a277425eec659] to archive 2024-11-18T03:17:14,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:17:14,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d15272a30aeb427fb9a3a67306c2d622 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d15272a30aeb427fb9a3a67306c2d622 2024-11-18T03:17:14,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/35de357400244c6797afed84dcd023f3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/35de357400244c6797afed84dcd023f3 2024-11-18T03:17:14,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ad7fecc4755e4c0f8f4cddf40cfc67a3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ad7fecc4755e4c0f8f4cddf40cfc67a3 2024-11-18T03:17:14,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e078efde167b44218ed1ce66e7ffb99a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e078efde167b44218ed1ce66e7ffb99a 2024-11-18T03:17:14,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/8f4982741f8d43edb380d1afa1351217 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/8f4982741f8d43edb380d1afa1351217 2024-11-18T03:17:14,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2358409e02454d7ab000452fe12c0f7c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2358409e02454d7ab000452fe12c0f7c 2024-11-18T03:17:14,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/42b12ef709334691be68848204b73743 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/42b12ef709334691be68848204b73743 2024-11-18T03:17:14,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7220092a4f584bec81120ef0cf5b80ea to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7220092a4f584bec81120ef0cf5b80ea 2024-11-18T03:17:14,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c1de0445bb6d4529bb301529164a7346 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c1de0445bb6d4529bb301529164a7346 2024-11-18T03:17:14,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2c9acecf5d4d44f7b56edd886f75dc48 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/2c9acecf5d4d44f7b56edd886f75dc48 2024-11-18T03:17:14,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/92ea590ddfa44c8f9f97769bf4179a08 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/92ea590ddfa44c8f9f97769bf4179a08 2024-11-18T03:17:14,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/84fe4196fcaa42f0a8f102596955cdb4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/84fe4196fcaa42f0a8f102596955cdb4 2024-11-18T03:17:14,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6f60670d40bf4ae9ae2d342e5ea96bfe to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6f60670d40bf4ae9ae2d342e5ea96bfe 2024-11-18T03:17:14,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/9db474b8a2154c828888133124b05ade to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/9db474b8a2154c828888133124b05ade 2024-11-18T03:17:14,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/4f29622273dc4747a1b420f7217387b7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/4f29622273dc4747a1b420f7217387b7 2024-11-18T03:17:14,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/a9a9f9cad1154a33be2a961b1b3e38f8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/a9a9f9cad1154a33be2a961b1b3e38f8 2024-11-18T03:17:14,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/85bd25a9c7244f12a510fc5600aa49eb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/85bd25a9c7244f12a510fc5600aa49eb 2024-11-18T03:17:14,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e0d4ca8bb2f24ec0a4de870cd6f268dc to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e0d4ca8bb2f24ec0a4de870cd6f268dc 2024-11-18T03:17:14,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d4f7084200d745b095b1d939a0ae85a4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/d4f7084200d745b095b1d939a0ae85a4 2024-11-18T03:17:14,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ba64f419f79343dab2c8690a3942bdca to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ba64f419f79343dab2c8690a3942bdca 2024-11-18T03:17:14,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c13c980c386144b6a4234e07fb837635 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c13c980c386144b6a4234e07fb837635 2024-11-18T03:17:14,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7c7481370f2449808586175b63a51ad9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/7c7481370f2449808586175b63a51ad9 2024-11-18T03:17:14,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/edc04a647a1842458504978c1beee867 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/edc04a647a1842458504978c1beee867 2024-11-18T03:17:14,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ca089b29e4a744019f2cbcc6ce88d744 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/ca089b29e4a744019f2cbcc6ce88d744 2024-11-18T03:17:14,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/096bd4a8fe2f42afb5c50ccbb3a7ed27 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/096bd4a8fe2f42afb5c50ccbb3a7ed27 2024-11-18T03:17:14,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/f6669744a764491b9b5c9affcc7cd461 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/f6669744a764491b9b5c9affcc7cd461 2024-11-18T03:17:14,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/cf478e97dba64c48ab594856d77c1cee to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/cf478e97dba64c48ab594856d77c1cee 2024-11-18T03:17:14,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/5d13da8b0e08478fb4a479e6fc8e49b7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/5d13da8b0e08478fb4a479e6fc8e49b7 2024-11-18T03:17:14,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e09aa98e48f445f389250247d30a2d42 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/e09aa98e48f445f389250247d30a2d42 2024-11-18T03:17:14,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6762a651870f4e6b80c87678b72f552e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/6762a651870f4e6b80c87678b72f552e 2024-11-18T03:17:14,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c703621071254c949c9a277425eec659 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/c703621071254c949c9a277425eec659 2024-11-18T03:17:14,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7480a22614804c0b83edea2caf80a752, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5994f4c46d54428b8df0f2a8c8eb4455, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/97247aabf2ba444699a7d460e8339c1f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/169c1be4a488482f9151652780309373, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f1a37115bc60467ea96bc86943064cfc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/567c072ab10e468097271508cbab4f38, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9c4f10f7fe1b4b2babe491c2cffc47b4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/289376415cb8469e822156c08d34d11b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/a7b620ee52e243b5bf2aa695eaf21ae3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6baab0b2af94522862d4e80150c47f1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/505bb7114be5416bbb80ef24db8469b5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fcb4be2a8a9c483fb5c5061d6a439bb0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/00e78d2b0c634d73950bbc600a2cb0e7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7e722a22ed6245cab8d4a2d618238d64, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f081289136f6414a9305961438aad1a3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7b5f01fddbf24e119592e53220699c47, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b42514354c194d8a85c63b0dbbcb3aff, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/cbc690c13aec40a9b009b82d89cd1fe4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d9148472c89744af8f6ff3d26553bc6d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/245eb8a952904e98affae9472ff2662b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/81cd49f026df4ff0836373f0768e650d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5ae45f5ec26449a4b7401026e230b95d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/1f83ecbbaabc4ef6aa4c69eac6bc329e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/3a86e916d2a54b848d68344aff49e2a2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b49813c7d7a64cd391e465d69f18fc2a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6516ea439a448248afa629f637b7f6e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/de72de6806ef4829a3e0f9bc0773c545, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fdd15fb5a72d47d6b9b99bd014c83e04, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/94be603224374d22bdf4a88a67d50706, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/23c423d69d924b36ac64422bd642ebd9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/65e8e111f6a247ffbc2beedf33264ab0] to archive 2024-11-18T03:17:14,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:17:14,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7480a22614804c0b83edea2caf80a752 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7480a22614804c0b83edea2caf80a752 2024-11-18T03:17:14,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5994f4c46d54428b8df0f2a8c8eb4455 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5994f4c46d54428b8df0f2a8c8eb4455 2024-11-18T03:17:14,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/97247aabf2ba444699a7d460e8339c1f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/97247aabf2ba444699a7d460e8339c1f 2024-11-18T03:17:14,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/169c1be4a488482f9151652780309373 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/169c1be4a488482f9151652780309373 2024-11-18T03:17:14,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f1a37115bc60467ea96bc86943064cfc to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f1a37115bc60467ea96bc86943064cfc 2024-11-18T03:17:14,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/567c072ab10e468097271508cbab4f38 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/567c072ab10e468097271508cbab4f38 2024-11-18T03:17:14,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9c4f10f7fe1b4b2babe491c2cffc47b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9c4f10f7fe1b4b2babe491c2cffc47b4 2024-11-18T03:17:14,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/289376415cb8469e822156c08d34d11b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/289376415cb8469e822156c08d34d11b 2024-11-18T03:17:14,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/a7b620ee52e243b5bf2aa695eaf21ae3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/a7b620ee52e243b5bf2aa695eaf21ae3 2024-11-18T03:17:14,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6baab0b2af94522862d4e80150c47f1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6baab0b2af94522862d4e80150c47f1 2024-11-18T03:17:14,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/505bb7114be5416bbb80ef24db8469b5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/505bb7114be5416bbb80ef24db8469b5 2024-11-18T03:17:14,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fcb4be2a8a9c483fb5c5061d6a439bb0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fcb4be2a8a9c483fb5c5061d6a439bb0 2024-11-18T03:17:14,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/00e78d2b0c634d73950bbc600a2cb0e7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/00e78d2b0c634d73950bbc600a2cb0e7 2024-11-18T03:17:14,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7e722a22ed6245cab8d4a2d618238d64 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7e722a22ed6245cab8d4a2d618238d64 2024-11-18T03:17:14,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f081289136f6414a9305961438aad1a3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/f081289136f6414a9305961438aad1a3 2024-11-18T03:17:14,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7b5f01fddbf24e119592e53220699c47 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/7b5f01fddbf24e119592e53220699c47 2024-11-18T03:17:14,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b42514354c194d8a85c63b0dbbcb3aff to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b42514354c194d8a85c63b0dbbcb3aff 2024-11-18T03:17:14,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/cbc690c13aec40a9b009b82d89cd1fe4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/cbc690c13aec40a9b009b82d89cd1fe4 2024-11-18T03:17:14,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d9148472c89744af8f6ff3d26553bc6d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d9148472c89744af8f6ff3d26553bc6d 2024-11-18T03:17:14,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/245eb8a952904e98affae9472ff2662b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/245eb8a952904e98affae9472ff2662b 2024-11-18T03:17:14,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/81cd49f026df4ff0836373f0768e650d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/81cd49f026df4ff0836373f0768e650d 2024-11-18T03:17:14,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5ae45f5ec26449a4b7401026e230b95d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/5ae45f5ec26449a4b7401026e230b95d 2024-11-18T03:17:14,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/1f83ecbbaabc4ef6aa4c69eac6bc329e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/1f83ecbbaabc4ef6aa4c69eac6bc329e 2024-11-18T03:17:14,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/3a86e916d2a54b848d68344aff49e2a2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/3a86e916d2a54b848d68344aff49e2a2 2024-11-18T03:17:14,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b49813c7d7a64cd391e465d69f18fc2a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b49813c7d7a64cd391e465d69f18fc2a 2024-11-18T03:17:14,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6516ea439a448248afa629f637b7f6e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/b6516ea439a448248afa629f637b7f6e 2024-11-18T03:17:14,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/de72de6806ef4829a3e0f9bc0773c545 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/de72de6806ef4829a3e0f9bc0773c545 2024-11-18T03:17:14,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fdd15fb5a72d47d6b9b99bd014c83e04 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/fdd15fb5a72d47d6b9b99bd014c83e04 2024-11-18T03:17:14,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/94be603224374d22bdf4a88a67d50706 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/94be603224374d22bdf4a88a67d50706 2024-11-18T03:17:14,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/23c423d69d924b36ac64422bd642ebd9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/23c423d69d924b36ac64422bd642ebd9 2024-11-18T03:17:14,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/65e8e111f6a247ffbc2beedf33264ab0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/65e8e111f6a247ffbc2beedf33264ab0 2024-11-18T03:17:14,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/787f18e0409e485a8e0e2e24c508cf77, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/1b15d95439234d0aa2f3be16e384816e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd30d6b7c7034fd39e400eed69a87586, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/354c2e8a6743416ba2209d45e7e74d95, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd941ec2765c45aba829008ab6b6d9c9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ceaec43ca4fa4720915ed56aab5a8484, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/6274b695631049b3b73254d4a3e4df20, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/11381eaf3f904b64a8ca55b867add888, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/7225dc2523664fb39b2d2f2cee1b32ee, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/db9517a620534f9aa44ffe76d89410eb, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/09c6d0856f7449809d0bc5808556696a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0a9104df59d04a37918377545f7fe91e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9727845528ff438fa1c3c49596dc4e92, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ebcff4a49ae49b989a88c7b09935a93, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f304f3cbe54b435585d7e6da1b1fea23, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/090e80391ddc4c2187aaedb5a4fe3b29, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ab37197c770742f1990b185b630c2168, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/dbe532db466a41d294b039d02eea0fed, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ae39a2675c0e426ca6499447e3d9bbd1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/aa467a18666340a4bb6e8068db5c5541, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/983425afbfc64dd18ea140daf6bc4b42, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/2f0939e9ff5d422a8c6e865f1342ba34, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ec2c847fbed48b7b4584f4e1ff46ec3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c11be004a0bf43d08bf18bb2f54a2192, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f0905435da8b454ab6e5550200156c95, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/adce04966d1944d78f0e3730965134f2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c0c291619d28442b8da65918b68ac57e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/305ab37facab41eca2c67f1620c76bd9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/65a4485459f7437180a092f638f0a3e3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ff892103e1944c01852aefa37534bca7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/a1affa1c41e243a889d186ffd0b46d0a] to archive 2024-11-18T03:17:14,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:17:14,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/787f18e0409e485a8e0e2e24c508cf77 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/787f18e0409e485a8e0e2e24c508cf77 2024-11-18T03:17:14,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/1b15d95439234d0aa2f3be16e384816e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/1b15d95439234d0aa2f3be16e384816e 2024-11-18T03:17:14,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd30d6b7c7034fd39e400eed69a87586 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd30d6b7c7034fd39e400eed69a87586 2024-11-18T03:17:14,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/354c2e8a6743416ba2209d45e7e74d95 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/354c2e8a6743416ba2209d45e7e74d95 2024-11-18T03:17:14,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd941ec2765c45aba829008ab6b6d9c9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/cd941ec2765c45aba829008ab6b6d9c9 2024-11-18T03:17:14,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ceaec43ca4fa4720915ed56aab5a8484 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ceaec43ca4fa4720915ed56aab5a8484 2024-11-18T03:17:14,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/6274b695631049b3b73254d4a3e4df20 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/6274b695631049b3b73254d4a3e4df20 2024-11-18T03:17:14,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/11381eaf3f904b64a8ca55b867add888 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/11381eaf3f904b64a8ca55b867add888 2024-11-18T03:17:14,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/7225dc2523664fb39b2d2f2cee1b32ee to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/7225dc2523664fb39b2d2f2cee1b32ee 2024-11-18T03:17:14,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/db9517a620534f9aa44ffe76d89410eb to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/db9517a620534f9aa44ffe76d89410eb 2024-11-18T03:17:14,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/09c6d0856f7449809d0bc5808556696a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/09c6d0856f7449809d0bc5808556696a 2024-11-18T03:17:14,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0a9104df59d04a37918377545f7fe91e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0a9104df59d04a37918377545f7fe91e 2024-11-18T03:17:14,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9727845528ff438fa1c3c49596dc4e92 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9727845528ff438fa1c3c49596dc4e92 2024-11-18T03:17:14,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ebcff4a49ae49b989a88c7b09935a93 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ebcff4a49ae49b989a88c7b09935a93 2024-11-18T03:17:14,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f304f3cbe54b435585d7e6da1b1fea23 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f304f3cbe54b435585d7e6da1b1fea23 2024-11-18T03:17:14,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/090e80391ddc4c2187aaedb5a4fe3b29 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/090e80391ddc4c2187aaedb5a4fe3b29 2024-11-18T03:17:14,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ab37197c770742f1990b185b630c2168 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ab37197c770742f1990b185b630c2168 2024-11-18T03:17:14,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/dbe532db466a41d294b039d02eea0fed to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/dbe532db466a41d294b039d02eea0fed 2024-11-18T03:17:14,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ae39a2675c0e426ca6499447e3d9bbd1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ae39a2675c0e426ca6499447e3d9bbd1 2024-11-18T03:17:14,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/aa467a18666340a4bb6e8068db5c5541 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/aa467a18666340a4bb6e8068db5c5541 2024-11-18T03:17:14,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/983425afbfc64dd18ea140daf6bc4b42 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/983425afbfc64dd18ea140daf6bc4b42 2024-11-18T03:17:14,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/2f0939e9ff5d422a8c6e865f1342ba34 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/2f0939e9ff5d422a8c6e865f1342ba34 2024-11-18T03:17:14,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ec2c847fbed48b7b4584f4e1ff46ec3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/9ec2c847fbed48b7b4584f4e1ff46ec3 2024-11-18T03:17:14,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c11be004a0bf43d08bf18bb2f54a2192 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c11be004a0bf43d08bf18bb2f54a2192 2024-11-18T03:17:14,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f0905435da8b454ab6e5550200156c95 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/f0905435da8b454ab6e5550200156c95 2024-11-18T03:17:14,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/adce04966d1944d78f0e3730965134f2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/adce04966d1944d78f0e3730965134f2 2024-11-18T03:17:14,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c0c291619d28442b8da65918b68ac57e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/c0c291619d28442b8da65918b68ac57e 2024-11-18T03:17:14,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/305ab37facab41eca2c67f1620c76bd9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/305ab37facab41eca2c67f1620c76bd9 2024-11-18T03:17:14,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/65a4485459f7437180a092f638f0a3e3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/65a4485459f7437180a092f638f0a3e3 2024-11-18T03:17:14,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ff892103e1944c01852aefa37534bca7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/ff892103e1944c01852aefa37534bca7 2024-11-18T03:17:14,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/a1affa1c41e243a889d186ffd0b46d0a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/a1affa1c41e243a889d186ffd0b46d0a 2024-11-18T03:17:14,782 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/recovered.edits/414.seqid, newMaxSeqId=414, maxSeqId=1 2024-11-18T03:17:14,783 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. 2024-11-18T03:17:14,783 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 9d402698952bfb73679bd006311c3148: Waiting for close lock at 1731899834124Running coprocessor pre-close hooks at 1731899834124Disabling compacts and flushes for region at 1731899834124Disabling writes for close at 1731899834124Obtaining lock to block concurrent updates at 1731899834124Preparing flush snapshotting stores in 9d402698952bfb73679bd006311c3148 at 1731899834124Finished memstore snapshotting TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731899834125 (+1 ms)Flushing stores of TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148. at 1731899834125Flushing 9d402698952bfb73679bd006311c3148/A: creating writer at 1731899834125Flushing 9d402698952bfb73679bd006311c3148/A: appending metadata at 1731899834128 (+3 ms)Flushing 9d402698952bfb73679bd006311c3148/A: closing flushed file at 1731899834128Flushing 9d402698952bfb73679bd006311c3148/B: creating writer at 1731899834153 (+25 ms)Flushing 9d402698952bfb73679bd006311c3148/B: appending metadata at 1731899834156 (+3 ms)Flushing 9d402698952bfb73679bd006311c3148/B: closing flushed file at 1731899834156Flushing 9d402698952bfb73679bd006311c3148/C: creating writer at 1731899834170 (+14 ms)Flushing 9d402698952bfb73679bd006311c3148/C: appending metadata at 1731899834173 (+3 ms)Flushing 9d402698952bfb73679bd006311c3148/C: closing flushed file at 1731899834173Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57910c65: reopening flushed file at 1731899834583 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44cdfc99: reopening flushed file at 1731899834588 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1298c3e6: reopening flushed file at 1731899834593 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 9d402698952bfb73679bd006311c3148 in 474ms, sequenceid=411, compaction requested=false at 1731899834598 (+5 ms)Writing region close event to WAL at 1731899834776 (+178 ms)Running coprocessor post-close hooks at 1731899834782 (+6 ms)Closed at 1731899834783 (+1 ms) 2024-11-18T03:17:14,785 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 9d402698952bfb73679bd006311c3148 2024-11-18T03:17:14,786 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=9d402698952bfb73679bd006311c3148, regionState=CLOSED 2024-11-18T03:17:14,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:17:14,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-18T03:17:14,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d402698952bfb73679bd006311c3148, server=fd862c3a04f9,38549,1731899684479 in 819 msec 2024-11-18T03:17:14,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-18T03:17:14,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d402698952bfb73679bd006311c3148, UNASSIGN in 825 msec 2024-11-18T03:17:14,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-18T03:17:14,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 829 msec 2024-11-18T03:17:14,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899834797"}]},"ts":"1731899834797"} 2024-11-18T03:17:14,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T03:17:14,799 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T03:17:14,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 845 msec 2024-11-18T03:17:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T03:17:15,102 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:15,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-18T03:17:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:15,105 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:15,106 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T03:17:15,108 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148 2024-11-18T03:17:15,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/recovered.edits] 2024-11-18T03:17:15,120 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/26d5f64e6ce048c88f036da4db3bcba7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/26d5f64e6ce048c88f036da4db3bcba7 2024-11-18T03:17:15,121 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/54560340eeca4dcb936f5e046aa124ca to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/A/54560340eeca4dcb936f5e046aa124ca 2024-11-18T03:17:15,124 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9eca1519f819499298c0e46137e8536e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/9eca1519f819499298c0e46137e8536e 2024-11-18T03:17:15,128 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d2f430a6bf374ab0a2e04dfd5845639e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/B/d2f430a6bf374ab0a2e04dfd5845639e 2024-11-18T03:17:15,130 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0d81a4a90c22402ea3d150b9fd047872 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/0d81a4a90c22402ea3d150b9fd047872 2024-11-18T03:17:15,131 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/b5630a12d1c84c16975a07819816b0ea to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/C/b5630a12d1c84c16975a07819816b0ea 2024-11-18T03:17:15,135 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/recovered.edits/414.seqid to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148/recovered.edits/414.seqid 2024-11-18T03:17:15,135 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/9d402698952bfb73679bd006311c3148 2024-11-18T03:17:15,135 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T03:17:15,138 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:15,140 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T03:17:15,142 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T03:17:15,143 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:15,143 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T03:17:15,144 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731899835143"}]},"ts":"9223372036854775807"} 2024-11-18T03:17:15,146 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T03:17:15,146 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9d402698952bfb73679bd006311c3148, NAME => 'TestAcidGuarantees,,1731899808829.9d402698952bfb73679bd006311c3148.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T03:17:15,146 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T03:17:15,146 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731899835146"}]},"ts":"9223372036854775807"} 2024-11-18T03:17:15,149 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T03:17:15,150 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:15,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-18T03:17:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T03:17:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T03:17:15,214 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:15,223 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 247) - Thread LEAK? -, OpenFileDescriptor=457 (was 460), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1274 (was 997) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1781 (was 2306) 2024-11-18T03:17:15,234 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1274, ProcessCount=11, AvailableMemoryMB=1781 2024-11-18T03:17:15,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:17:15,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:17:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:15,238 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:17:15,239 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:15,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-18T03:17:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T03:17:15,240 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:17:15,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742402_1578 (size=1061) 2024-11-18T03:17:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T03:17:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T03:17:15,647 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1 2024-11-18T03:17:15,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742403_1579 (size=53) 2024-11-18T03:17:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T03:17:16,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:17:16,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 66ab176d63aa2fde549e454fab5778b4, disabling compactions & flushes 2024-11-18T03:17:16,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. after waiting 0 ms 2024-11-18T03:17:16,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66ab176d63aa2fde549e454fab5778b4: Waiting for close lock at 1731899836077Disabling compacts and flushes for region at 1731899836077Disabling writes for close at 1731899836077Writing region close event to WAL at 1731899836077Closed at 1731899836077 2024-11-18T03:17:16,079 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:17:16,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731899836079"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731899836079"}]},"ts":"1731899836079"} 2024-11-18T03:17:16,081 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:17:16,082 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:17:16,082 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899836082"}]},"ts":"1731899836082"} 2024-11-18T03:17:16,084 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T03:17:16,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, ASSIGN}] 2024-11-18T03:17:16,086 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, ASSIGN 2024-11-18T03:17:16,086 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, ASSIGN; state=OFFLINE, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=false 2024-11-18T03:17:16,237 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:16,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, ASSIGN because future has completed 2024-11-18T03:17:16,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:17:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T03:17:16,394 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,394 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:17:16,394 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,394 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:17:16,395 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,395 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,396 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,397 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:17:16,398 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66ab176d63aa2fde549e454fab5778b4 columnFamilyName A 2024-11-18T03:17:16,398 DEBUG [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:16,398 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(327): Store=66ab176d63aa2fde549e454fab5778b4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:17:16,398 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,399 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:17:16,400 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66ab176d63aa2fde549e454fab5778b4 columnFamilyName B 2024-11-18T03:17:16,400 DEBUG [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:16,400 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(327): Store=66ab176d63aa2fde549e454fab5778b4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:17:16,400 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,402 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:17:16,402 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66ab176d63aa2fde549e454fab5778b4 columnFamilyName C 2024-11-18T03:17:16,402 DEBUG [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:16,402 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(327): Store=66ab176d63aa2fde549e454fab5778b4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:17:16,403 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,403 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,403 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,404 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,406 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,406 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,406 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:17:16,408 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,411 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:17:16,411 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 66ab176d63aa2fde549e454fab5778b4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70892281, jitterRate=0.05637730658054352}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:17:16,411 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:16,412 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 66ab176d63aa2fde549e454fab5778b4: Running coprocessor pre-open hook at 1731899836395Writing region info on filesystem at 1731899836395Initializing all the Stores at 1731899836395Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899836395Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899836396 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899836396Cleaning up temporary data from old regions at 1731899836406 (+10 ms)Running coprocessor post-open hooks at 1731899836411 (+5 ms)Region opened successfully at 1731899836412 (+1 ms) 2024-11-18T03:17:16,413 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., pid=142, masterSystemTime=1731899836390 2024-11-18T03:17:16,415 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,415 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:16,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=OPEN, openSeqNum=2, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:16,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:17:16,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-18T03:17:16,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 in 181 msec 2024-11-18T03:17:16,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-18T03:17:16,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, ASSIGN in 337 msec 2024-11-18T03:17:16,426 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:17:16,426 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899836426"}]},"ts":"1731899836426"} 2024-11-18T03:17:16,428 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T03:17:16,429 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:17:16,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-18T03:17:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T03:17:17,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:17,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d71c2a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:17,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:17,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:17,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aec509a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:17,397 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3718c858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:17,398 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:17,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:17,401 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:17:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fd862c3a04f9,45051,1731899683325 2024-11-18T03:17:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7cc5353f 2024-11-18T03:17:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:17:17,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:17:17,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T03:17:17,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:17:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:17,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742404_1580 (size=1097) 2024-11-18T03:17:17,821 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T03:17:17,821 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T03:17:17,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:17:17,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, REOPEN/MOVE}] 2024-11-18T03:17:17,828 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, REOPEN/MOVE 2024-11-18T03:17:17,829 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:17,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, REOPEN/MOVE because future has completed 2024-11-18T03:17:17,831 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:17:17,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:17:17,993 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:17,993 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:17:17,993 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 66ab176d63aa2fde549e454fab5778b4, disabling compactions & flushes 2024-11-18T03:17:17,993 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:17,993 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:17,993 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. after waiting 0 ms 2024-11-18T03:17:17,993 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:17,997 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T03:17:17,997 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:17,997 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 66ab176d63aa2fde549e454fab5778b4: Waiting for close lock at 1731899837993Running coprocessor pre-close hooks at 1731899837993Disabling compacts and flushes for region at 1731899837993Disabling writes for close at 1731899837993Writing region close event to WAL at 1731899837994 (+1 ms)Running coprocessor post-close hooks at 1731899837997 (+3 ms)Closed at 1731899837997 2024-11-18T03:17:17,997 WARN [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 66ab176d63aa2fde549e454fab5778b4 to self. 2024-11-18T03:17:17,999 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,000 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=CLOSED 2024-11-18T03:17:18,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:17:18,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-18T03:17:18,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 in 172 msec 2024-11-18T03:17:18,005 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, REOPEN/MOVE; state=CLOSED, location=fd862c3a04f9,38549,1731899684479; forceNewPlan=false, retain=true 2024-11-18T03:17:18,156 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=OPENING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, REOPEN/MOVE because future has completed 2024-11-18T03:17:18,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:17:18,312 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,313 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:17:18,313 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,313 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:17:18,313 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,313 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,314 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,315 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:17:18,315 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66ab176d63aa2fde549e454fab5778b4 columnFamilyName A 2024-11-18T03:17:18,316 DEBUG [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:18,317 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(327): Store=66ab176d63aa2fde549e454fab5778b4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:17:18,317 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,318 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:17:18,318 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66ab176d63aa2fde549e454fab5778b4 columnFamilyName B 2024-11-18T03:17:18,318 DEBUG [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:18,318 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(327): Store=66ab176d63aa2fde549e454fab5778b4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:17:18,318 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,319 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T03:17:18,319 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66ab176d63aa2fde549e454fab5778b4 columnFamilyName C 2024-11-18T03:17:18,319 DEBUG [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:18,319 INFO [StoreOpener-66ab176d63aa2fde549e454fab5778b4-1 {}] regionserver.HStore(327): Store=66ab176d63aa2fde549e454fab5778b4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:17:18,319 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,319 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,320 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,321 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,322 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,322 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,322 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T03:17:18,324 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,324 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 66ab176d63aa2fde549e454fab5778b4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71279513, jitterRate=0.062147513031959534}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T03:17:18,324 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,325 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 66ab176d63aa2fde549e454fab5778b4: Running coprocessor pre-open hook at 1731899838313Writing region info on filesystem at 1731899838313Initializing all the Stores at 1731899838314 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899838314Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899838314Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731899838314Cleaning up temporary data from old regions at 1731899838322 (+8 ms)Running coprocessor post-open hooks at 1731899838324 (+2 ms)Region opened successfully at 1731899838325 (+1 ms) 2024-11-18T03:17:18,326 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., pid=147, masterSystemTime=1731899838310 2024-11-18T03:17:18,328 DEBUG [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,328 INFO [RS_OPEN_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=OPEN, openSeqNum=5, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:17:18,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-18T03:17:18,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 in 174 msec 2024-11-18T03:17:18,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-18T03:17:18,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, REOPEN/MOVE in 508 msec 2024-11-18T03:17:18,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-18T03:17:18,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-11-18T03:17:18,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-11-18T03:17:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-18T03:17:18,340 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:18,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ecb8793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,342 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2301fd2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac931ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccad3c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,346 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ffaf90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,348 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a78a46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74abaaed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,350 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14a13639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5cb605, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55149d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4bae20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,355 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa10f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70123c26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,357 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562d627a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,358 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10be988f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3499f302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,360 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f2cfd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f16eaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1675bdc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,363 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f746ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@560fff51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78526251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,368 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158d97fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,369 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df6da07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc4680c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,371 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,372 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff9458c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fe620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fd862c3a04f9,45051,-1 for getting cluster id 2024-11-18T03:17:18,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:17:18,374 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd21d32cf-9d00-4e6e-954c-ada723a6fe6a' 2024-11-18T03:17:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:17:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d21d32cf-9d00-4e6e-954c-ada723a6fe6a" 2024-11-18T03:17:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eec040d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fd862c3a04f9,45051,-1] 2024-11-18T03:17:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:17:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:18,375 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:17:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7798e3cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:17:18,376 DEBUG [Thread-2580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,378 DEBUG [Thread-2573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,378 DEBUG [Thread-2575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,379 DEBUG [Thread-2577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,379 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-18T03:17:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,381 DEBUG [Thread-2582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T03:17:18,382 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:18,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,382 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:18,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,389 DEBUG [Thread-2586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T03:17:18,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:18,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:18,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:18,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:18,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:18,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:18,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,396 DEBUG [Thread-2584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,397 DEBUG [Thread-2569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,397 DEBUG [Thread-2571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,397 DEBUG [Thread-2588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:17:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fd862c3a04f9,38549,1731899684479, seqNum=-1] 2024-11-18T03:17:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:17:18,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:17:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5] 2024-11-18T03:17:18,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118638399e09548474d8443d50f530fe43b_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_1/A:col10/1731899838389/Put/seqid=0 2024-11-18T03:17:18,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899838433, max timestamp=1731899838438Num uniques 859; 2024-11-18T03:17:18,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:18,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899838433, max timestamp=1731899838438Num uniques 859; 2024-11-18T03:17:18,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899838433, max timestamp=1731899838438Num uniques 859; 2024-11-18T03:17:18,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:18,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:18,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742405_1581 (size=9714) 2024-11-18T03:17:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T03:17:18,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899898515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899898517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899898517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899898518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899898522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T03:17:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899898632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899898632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899898632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899898632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899898632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T03:17:18,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T03:17:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899898840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899898840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899898841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899898841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899898841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:18,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T03:17:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:18,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:18,873 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:18,878 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118638399e09548474d8443d50f530fe43b_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118638399e09548474d8443d50f530fe43b_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:18,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/153e61cbd475463181aa0cf21c4d2225, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:18,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/153e61cbd475463181aa0cf21c4d2225 is 175, key is test_row_1/A:col10/1731899838389/Put/seqid=0 2024-11-18T03:17:18,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742406_1582 (size=22361) 2024-11-18T03:17:18,887 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/153e61cbd475463181aa0cf21c4d2225 2024-11-18T03:17:18,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/ed53ab5c9cfd47558cc77acc1007280a is 50, key is test_row_1/B:col10/1731899838389/Put/seqid=0 2024-11-18T03:17:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742407_1583 (size=9657) 2024-11-18T03:17:18,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/ed53ab5c9cfd47558cc77acc1007280a 2024-11-18T03:17:18,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/6a468398b9d445a0ab0bea3928f4087c is 50, key is test_row_1/C:col10/1731899838389/Put/seqid=0 2024-11-18T03:17:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742408_1584 (size=9657) 2024-11-18T03:17:19,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T03:17:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T03:17:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899899151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899899152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899899151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899899152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899899152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T03:17:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T03:17:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:19,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/6a468398b9d445a0ab0bea3928f4087c 2024-11-18T03:17:19,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/153e61cbd475463181aa0cf21c4d2225 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/153e61cbd475463181aa0cf21c4d2225 2024-11-18T03:17:19,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/153e61cbd475463181aa0cf21c4d2225, entries=100, sequenceid=15, filesize=21.8 K 2024-11-18T03:17:19,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/ed53ab5c9cfd47558cc77acc1007280a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ed53ab5c9cfd47558cc77acc1007280a 2024-11-18T03:17:19,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ed53ab5c9cfd47558cc77acc1007280a, entries=100, sequenceid=15, filesize=9.4 K 2024-11-18T03:17:19,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/6a468398b9d445a0ab0bea3928f4087c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6a468398b9d445a0ab0bea3928f4087c 2024-11-18T03:17:19,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6a468398b9d445a0ab0bea3928f4087c, entries=100, sequenceid=15, filesize=9.4 K 2024-11-18T03:17:19,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 66ab176d63aa2fde549e454fab5778b4 in 978ms, sequenceid=15, compaction requested=false 2024-11-18T03:17:19,369 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T03:17:19,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:19,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T03:17:19,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,473 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T03:17:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fc8353c0b91b4ecb9515a760af93f95f_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899838514/Put/seqid=0 2024-11-18T03:17:19,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742409_1585 (size=12154) 2024-11-18T03:17:19,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:19,493 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fc8353c0b91b4ecb9515a760af93f95f_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fc8353c0b91b4ecb9515a760af93f95f_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f76375bb6ce043c29db4dae60dd05087, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:19,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f76375bb6ce043c29db4dae60dd05087 is 175, key is test_row_0/A:col10/1731899838514/Put/seqid=0 2024-11-18T03:17:19,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742410_1586 (size=30955) 2024-11-18T03:17:19,498 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f76375bb6ce043c29db4dae60dd05087 2024-11-18T03:17:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/5d1914bbfa7643eb802a90a7dbd5597b is 50, key is test_row_0/B:col10/1731899838514/Put/seqid=0 2024-11-18T03:17:19,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742411_1587 (size=12001) 2024-11-18T03:17:19,509 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/5d1914bbfa7643eb802a90a7dbd5597b 2024-11-18T03:17:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T03:17:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/94e4807c977d47e5a8920fe3a89d31f0 is 50, key is test_row_0/C:col10/1731899838514/Put/seqid=0 2024-11-18T03:17:19,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742412_1588 (size=12001) 2024-11-18T03:17:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:19,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899899665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899899667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899899667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899899670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899899673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899899782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899899782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899899782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899899782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899899784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:19,935 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/94e4807c977d47e5a8920fe3a89d31f0 2024-11-18T03:17:19,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f76375bb6ce043c29db4dae60dd05087 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f76375bb6ce043c29db4dae60dd05087 2024-11-18T03:17:19,944 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f76375bb6ce043c29db4dae60dd05087, entries=150, sequenceid=52, filesize=30.2 K 2024-11-18T03:17:19,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/5d1914bbfa7643eb802a90a7dbd5597b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5d1914bbfa7643eb802a90a7dbd5597b 2024-11-18T03:17:19,948 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5d1914bbfa7643eb802a90a7dbd5597b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-18T03:17:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/94e4807c977d47e5a8920fe3a89d31f0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/94e4807c977d47e5a8920fe3a89d31f0 2024-11-18T03:17:19,953 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/94e4807c977d47e5a8920fe3a89d31f0, entries=150, sequenceid=52, filesize=11.7 K 2024-11-18T03:17:19,954 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 66ab176d63aa2fde549e454fab5778b4 in 481ms, sequenceid=52, compaction requested=false 2024-11-18T03:17:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-18T03:17:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-18T03:17:19,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-18T03:17:19,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-18T03:17:19,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.5810 sec 2024-11-18T03:17:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:19,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T03:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:20,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189324b376d6754947903910b049a6b834_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899839992/Put/seqid=0 2024-11-18T03:17:20,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742413_1589 (size=14594) 2024-11-18T03:17:20,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899900016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731899840011, max timestamp=1731899840020Num uniques 859; 2024-11-18T03:17:20,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731899840011, max timestamp=1731899840020Num uniques 859; 2024-11-18T03:17:20,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T03:17:20,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T03:17:20,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731899840011, max timestamp=1731899840020Num uniques 859; 2024-11-18T03:17:20,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T03:17:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899900038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899900039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899900044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899900046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899900130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899900151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899900151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899900151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899900151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,313 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:17:20,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899900341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899900361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899900361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899900362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899900362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,408 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:20,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189324b376d6754947903910b049a6b834_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189324b376d6754947903910b049a6b834_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:20,418 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/de05e1c9f65d4a5599e3d8f9467960f7, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:20,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/de05e1c9f65d4a5599e3d8f9467960f7 is 175, key is test_row_0/A:col10/1731899839992/Put/seqid=0 2024-11-18T03:17:20,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742414_1590 (size=39549) 2024-11-18T03:17:20,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/de05e1c9f65d4a5599e3d8f9467960f7 2024-11-18T03:17:20,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/6392bbf3c6c04223b5bf931af125e3c1 is 50, key is test_row_0/B:col10/1731899839992/Put/seqid=0 2024-11-18T03:17:20,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742415_1591 (size=12001) 2024-11-18T03:17:20,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/6392bbf3c6c04223b5bf931af125e3c1 2024-11-18T03:17:20,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/5f9b6e42a25b4617b3c306cd4ccee75f is 50, key is test_row_0/C:col10/1731899839992/Put/seqid=0 2024-11-18T03:17:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T03:17:20,534 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:20,535 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:20,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742416_1592 (size=12001) 2024-11-18T03:17:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-18T03:17:20,538 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:20,539 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:20,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T03:17:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T03:17:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899900651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899900672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899900672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899900672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899900672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:20,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T03:17:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:20,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T03:17:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T03:17:20,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/5f9b6e42a25b4617b3c306cd4ccee75f 2024-11-18T03:17:20,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/de05e1c9f65d4a5599e3d8f9467960f7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/de05e1c9f65d4a5599e3d8f9467960f7 2024-11-18T03:17:20,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/de05e1c9f65d4a5599e3d8f9467960f7, entries=200, sequenceid=63, filesize=38.6 K 2024-11-18T03:17:20,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/6392bbf3c6c04223b5bf931af125e3c1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6392bbf3c6c04223b5bf931af125e3c1 2024-11-18T03:17:20,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6392bbf3c6c04223b5bf931af125e3c1, entries=150, sequenceid=63, filesize=11.7 K 2024-11-18T03:17:20,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/5f9b6e42a25b4617b3c306cd4ccee75f as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/5f9b6e42a25b4617b3c306cd4ccee75f 2024-11-18T03:17:20,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/5f9b6e42a25b4617b3c306cd4ccee75f, entries=150, sequenceid=63, filesize=11.7 K 2024-11-18T03:17:20,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 66ab176d63aa2fde549e454fab5778b4 in 956ms, sequenceid=63, compaction requested=true 2024-11-18T03:17:20,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:20,950 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:20,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:20,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:20,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:20,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:20,950 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:20,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:20,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:20,950 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:20,950 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:20,950 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,951 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/153e61cbd475463181aa0cf21c4d2225, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f76375bb6ce043c29db4dae60dd05087, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/de05e1c9f65d4a5599e3d8f9467960f7] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=90.7 K 2024-11-18T03:17:20,951 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/153e61cbd475463181aa0cf21c4d2225, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f76375bb6ce043c29db4dae60dd05087, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/de05e1c9f65d4a5599e3d8f9467960f7] 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 153e61cbd475463181aa0cf21c4d2225, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731899838387 2024-11-18T03:17:20,951 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,951 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ed53ab5c9cfd47558cc77acc1007280a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5d1914bbfa7643eb802a90a7dbd5597b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6392bbf3c6c04223b5bf931af125e3c1] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=32.9 K 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76375bb6ce043c29db4dae60dd05087, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731899838512 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ed53ab5c9cfd47558cc77acc1007280a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731899838387 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting de05e1c9f65d4a5599e3d8f9467960f7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899839663 2024-11-18T03:17:20,951 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1914bbfa7643eb802a90a7dbd5597b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731899838512 2024-11-18T03:17:20,952 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6392bbf3c6c04223b5bf931af125e3c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899839663 2024-11-18T03:17:20,958 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:20,962 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:20,962 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f3fe63f5ba7d4592ae64d9c73c9459f2 is 50, key is test_row_0/B:col10/1731899839992/Put/seqid=0 2024-11-18T03:17:20,964 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118d1d58f5fe25a42958d7ab301927c5a81_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:20,966 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118d1d58f5fe25a42958d7ab301927c5a81_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:20,966 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d1d58f5fe25a42958d7ab301927c5a81_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:20,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742417_1593 (size=12104) 2024-11-18T03:17:20,974 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f3fe63f5ba7d4592ae64d9c73c9459f2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f3fe63f5ba7d4592ae64d9c73c9459f2 2024-11-18T03:17:20,979 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into f3fe63f5ba7d4592ae64d9c73c9459f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:20,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:20,979 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=13, startTime=1731899840950; duration=0sec 2024-11-18T03:17:20,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:20,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:20,979 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:20,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742418_1594 (size=4469) 2024-11-18T03:17:20,981 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:20,981 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:20,981 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:20,981 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6a468398b9d445a0ab0bea3928f4087c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/94e4807c977d47e5a8920fe3a89d31f0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/5f9b6e42a25b4617b3c306cd4ccee75f] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=32.9 K 2024-11-18T03:17:20,982 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#504 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:20,982 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a468398b9d445a0ab0bea3928f4087c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731899838387 2024-11-18T03:17:20,982 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/dd6a50bc83324f87b515eade84de7af8 is 175, key is test_row_0/A:col10/1731899839992/Put/seqid=0 2024-11-18T03:17:20,983 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e4807c977d47e5a8920fe3a89d31f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731899838512 2024-11-18T03:17:20,983 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f9b6e42a25b4617b3c306cd4ccee75f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899839663 2024-11-18T03:17:21,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742419_1595 (size=31058) 2024-11-18T03:17:21,005 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:21,006 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e7f895c91ea7447faddecdd7783f8642 is 50, key is test_row_0/C:col10/1731899839992/Put/seqid=0 2024-11-18T03:17:21,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T03:17:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:21,007 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-18T03:17:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:21,012 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/dd6a50bc83324f87b515eade84de7af8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd6a50bc83324f87b515eade84de7af8 2024-11-18T03:17:21,018 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into dd6a50bc83324f87b515eade84de7af8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:21,018 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:21,018 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=13, startTime=1731899840949; duration=0sec 2024-11-18T03:17:21,018 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:21,018 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:21,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118179ebff8890f463f8d7de19299075ed0_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899840036/Put/seqid=0 2024-11-18T03:17:21,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742420_1596 (size=12104) 2024-11-18T03:17:21,044 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e7f895c91ea7447faddecdd7783f8642 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e7f895c91ea7447faddecdd7783f8642 2024-11-18T03:17:21,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742421_1597 (size=12154) 2024-11-18T03:17:21,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,049 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118179ebff8890f463f8d7de19299075ed0_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118179ebff8890f463f8d7de19299075ed0_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:21,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d26a2c74203440369a7ca6845c7c9aba, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:21,051 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into e7f895c91ea7447faddecdd7783f8642(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:21,051 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:21,051 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=13, startTime=1731899840950; duration=0sec 2024-11-18T03:17:21,051 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:21,051 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:21,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d26a2c74203440369a7ca6845c7c9aba is 175, key is test_row_0/A:col10/1731899840036/Put/seqid=0 2024-11-18T03:17:21,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742422_1598 (size=30955) 2024-11-18T03:17:21,081 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d26a2c74203440369a7ca6845c7c9aba 2024-11-18T03:17:21,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f7ab8971db3b4c93bf7f393e46072a95 is 50, key is test_row_0/B:col10/1731899840036/Put/seqid=0 2024-11-18T03:17:21,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742423_1599 (size=12001) 2024-11-18T03:17:21,133 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f7ab8971db3b4c93bf7f393e46072a95 2024-11-18T03:17:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e41de585d5dd484b9e5cfa4111ffaefe is 50, key is test_row_0/C:col10/1731899840036/Put/seqid=0 2024-11-18T03:17:21,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742424_1600 (size=12001) 2024-11-18T03:17:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:21,165 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e41de585d5dd484b9e5cfa4111ffaefe 2024-11-18T03:17:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d26a2c74203440369a7ca6845c7c9aba as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d26a2c74203440369a7ca6845c7c9aba 2024-11-18T03:17:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T03:17:21,175 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d26a2c74203440369a7ca6845c7c9aba, entries=150, sequenceid=98, filesize=30.2 K 2024-11-18T03:17:21,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f7ab8971db3b4c93bf7f393e46072a95 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f7ab8971db3b4c93bf7f393e46072a95 2024-11-18T03:17:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,182 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f7ab8971db3b4c93bf7f393e46072a95, entries=150, sequenceid=98, filesize=11.7 K 2024-11-18T03:17:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e41de585d5dd484b9e5cfa4111ffaefe as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e41de585d5dd484b9e5cfa4111ffaefe 2024-11-18T03:17:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,191 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e41de585d5dd484b9e5cfa4111ffaefe, entries=150, sequenceid=98, filesize=11.7 K 2024-11-18T03:17:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,192 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=26.84 KB/27480 for 66ab176d63aa2fde549e454fab5778b4 in 186ms, sequenceid=98, compaction requested=false 2024-11-18T03:17:21,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:21,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:21,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-18T03:17:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-18T03:17:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-18T03:17:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 661 msec 2024-11-18T03:17:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 668 msec 2024-11-18T03:17:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T03:17:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:21,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:21,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:21,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:21,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180c50dfa38fbf4ef1a01ae3cf6650aefb_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_1/A:col10/1731899841205/Put/seqid=0 2024-11-18T03:17:21,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731899841225, max timestamp=1731899841243Num uniques 859; 2024-11-18T03:17:21,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:17:21,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899841239, max timestamp=1731899841243Num uniques 859; 2024-11-18T03:17:21,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T03:17:21,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731899841239, max timestamp=1731899841243Num uniques 859; 2024-11-18T03:17:21,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T03:17:21,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742425_1601 (size=21918) 2024-11-18T03:17:21,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899901271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899901272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899901275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899901276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899901277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899901381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899901381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899901381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899901391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899901391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899901591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899901591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899901591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899901600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899901605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:21,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180c50dfa38fbf4ef1a01ae3cf6650aefb_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180c50dfa38fbf4ef1a01ae3cf6650aefb_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:21,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/73745087302d4d26bd20a95098c9317e, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:21,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/73745087302d4d26bd20a95098c9317e is 175, key is test_row_1/A:col10/1731899841205/Put/seqid=0 2024-11-18T03:17:21,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742426_1602 (size=65319) 2024-11-18T03:17:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T03:17:21,681 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:21,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-18T03:17:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T03:17:21,683 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:21,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:21,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T03:17:21,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T03:17:21,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:21,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:21,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:21,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:21,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899901900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899901900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899901900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899901911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899901921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:21,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T03:17:21,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:21,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:21,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:21,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:21,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T03:17:22,076 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/73745087302d4d26bd20a95098c9317e 2024-11-18T03:17:22,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/92a853056d1742bf8052c8140c68a309 is 50, key is test_row_1/B:col10/1731899841205/Put/seqid=0 2024-11-18T03:17:22,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742427_1603 (size=9657) 2024-11-18T03:17:22,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T03:17:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T03:17:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T03:17:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899902411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:22,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899902411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:22,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899902411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:22,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899902420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:22,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899902431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:22,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T03:17:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:22,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/92a853056d1742bf8052c8140c68a309 2024-11-18T03:17:22,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/459f13da0b074c708132cd82595dd5a4 is 50, key is test_row_1/C:col10/1731899841205/Put/seqid=0 2024-11-18T03:17:22,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742428_1604 (size=9657) 2024-11-18T03:17:22,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/459f13da0b074c708132cd82595dd5a4 2024-11-18T03:17:22,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/73745087302d4d26bd20a95098c9317e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/73745087302d4d26bd20a95098c9317e 2024-11-18T03:17:22,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/73745087302d4d26bd20a95098c9317e, entries=350, sequenceid=110, filesize=63.8 K 2024-11-18T03:17:22,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/92a853056d1742bf8052c8140c68a309 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/92a853056d1742bf8052c8140c68a309 2024-11-18T03:17:22,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/92a853056d1742bf8052c8140c68a309, entries=100, sequenceid=110, filesize=9.4 K 2024-11-18T03:17:22,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/459f13da0b074c708132cd82595dd5a4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/459f13da0b074c708132cd82595dd5a4 2024-11-18T03:17:22,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/459f13da0b074c708132cd82595dd5a4, entries=100, sequenceid=110, filesize=9.4 K 2024-11-18T03:17:22,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.15 KB/134300 for 66ab176d63aa2fde549e454fab5778b4 in 1331ms, sequenceid=110, compaction requested=true 2024-11-18T03:17:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:22,538 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:22,538 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:22,540 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:22,541 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,541 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f3fe63f5ba7d4592ae64d9c73c9459f2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f7ab8971db3b4c93bf7f393e46072a95, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/92a853056d1742bf8052c8140c68a309] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=33.0 K 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f3fe63f5ba7d4592ae64d9c73c9459f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899839663 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127332 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:22,541 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ab8971db3b4c93bf7f393e46072a95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731899840036 2024-11-18T03:17:22,541 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd6a50bc83324f87b515eade84de7af8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d26a2c74203440369a7ca6845c7c9aba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/73745087302d4d26bd20a95098c9317e] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=124.3 K 2024-11-18T03:17:22,541 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd6a50bc83324f87b515eade84de7af8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d26a2c74203440369a7ca6845c7c9aba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/73745087302d4d26bd20a95098c9317e] 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 92a853056d1742bf8052c8140c68a309, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731899841192 2024-11-18T03:17:22,541 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd6a50bc83324f87b515eade84de7af8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899839663 2024-11-18T03:17:22,542 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d26a2c74203440369a7ca6845c7c9aba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731899840036 2024-11-18T03:17:22,542 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73745087302d4d26bd20a95098c9317e, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731899841163 2024-11-18T03:17:22,548 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:22,549 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:22,549 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/a099cb973ed84b069490e96741f13784 is 50, key is test_row_0/B:col10/1731899840036/Put/seqid=0 2024-11-18T03:17:22,552 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118bf0ff79939d14e24b526cd89b26b178b_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:22,554 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118bf0ff79939d14e24b526cd89b26b178b_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:22,554 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bf0ff79939d14e24b526cd89b26b178b_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742429_1605 (size=12207) 2024-11-18T03:17:22,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742430_1606 (size=4469) 2024-11-18T03:17:22,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T03:17:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,600 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=131.15 KB heapSize=373.39 KB 2024-11-18T03:17:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:22,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c2f9146f0bdf421bbd1ec265120088e8_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899841271/Put/seqid=0 2024-11-18T03:17:22,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742431_1607 (size=12304) 2024-11-18T03:17:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T03:17:22,965 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:22,965 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/a099cb973ed84b069490e96741f13784 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a099cb973ed84b069490e96741f13784 2024-11-18T03:17:22,965 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/acb7ee0c4bc8452fbb32ed482ba10b36 is 175, key is test_row_0/A:col10/1731899840036/Put/seqid=0 2024-11-18T03:17:22,970 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into a099cb973ed84b069490e96741f13784(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:22,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:22,970 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=13, startTime=1731899842538; duration=0sec 2024-11-18T03:17:22,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:22,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:22,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:22,971 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:22,971 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:22,971 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:22,971 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e7f895c91ea7447faddecdd7783f8642, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e41de585d5dd484b9e5cfa4111ffaefe, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/459f13da0b074c708132cd82595dd5a4] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=33.0 K 2024-11-18T03:17:22,971 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e7f895c91ea7447faddecdd7783f8642, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731899839663 2024-11-18T03:17:22,971 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e41de585d5dd484b9e5cfa4111ffaefe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731899840036 2024-11-18T03:17:22,972 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 459f13da0b074c708132cd82595dd5a4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731899841192 2024-11-18T03:17:22,982 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:22,983 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/d0bdbe4e938749668e236c492b14fd90 is 50, key is test_row_0/C:col10/1731899840036/Put/seqid=0 2024-11-18T03:17:23,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742432_1608 (size=31268) 2024-11-18T03:17:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:23,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742433_1609 (size=12207) 2024-11-18T03:17:23,057 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c2f9146f0bdf421bbd1ec265120088e8_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c2f9146f0bdf421bbd1ec265120088e8_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:23,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ce133a22603c44828cf8e4a241594962, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:23,068 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/acb7ee0c4bc8452fbb32ed482ba10b36 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/acb7ee0c4bc8452fbb32ed482ba10b36 2024-11-18T03:17:23,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ce133a22603c44828cf8e4a241594962 is 175, key is test_row_0/A:col10/1731899841271/Put/seqid=0 2024-11-18T03:17:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742434_1610 (size=31105) 2024-11-18T03:17:23,120 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ce133a22603c44828cf8e4a241594962 2024-11-18T03:17:23,125 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into acb7ee0c4bc8452fbb32ed482ba10b36(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:23,126 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:23,126 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=13, startTime=1731899842538; duration=0sec 2024-11-18T03:17:23,126 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:23,126 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:23,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/67b5613f193e471b8ea38b39b6ba4c3a is 50, key is test_row_0/B:col10/1731899841271/Put/seqid=0 2024-11-18T03:17:23,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742435_1611 (size=12151) 2024-11-18T03:17:23,202 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/67b5613f193e471b8ea38b39b6ba4c3a 2024-11-18T03:17:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/305c4eedd62e41cea6fd93c964939dfa is 50, key is test_row_0/C:col10/1731899841271/Put/seqid=0 2024-11-18T03:17:23,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742436_1612 (size=12151) 2024-11-18T03:17:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:23,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:23,455 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/d0bdbe4e938749668e236c492b14fd90 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/d0bdbe4e938749668e236c492b14fd90 2024-11-18T03:17:23,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899903448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,461 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into d0bdbe4e938749668e236c492b14fd90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:23,461 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:23,461 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=13, startTime=1731899842538; duration=0sec 2024-11-18T03:17:23,461 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:23,461 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:23,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899903454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899903457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899903458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899903469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899903572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899903572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899903572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899903573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899903592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,654 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/305c4eedd62e41cea6fd93c964939dfa 2024-11-18T03:17:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ce133a22603c44828cf8e4a241594962 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ce133a22603c44828cf8e4a241594962 2024-11-18T03:17:23,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ce133a22603c44828cf8e4a241594962, entries=150, sequenceid=146, filesize=30.4 K 2024-11-18T03:17:23,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/67b5613f193e471b8ea38b39b6ba4c3a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/67b5613f193e471b8ea38b39b6ba4c3a 2024-11-18T03:17:23,707 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/67b5613f193e471b8ea38b39b6ba4c3a, entries=150, sequenceid=146, filesize=11.9 K 2024-11-18T03:17:23,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/305c4eedd62e41cea6fd93c964939dfa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/305c4eedd62e41cea6fd93c964939dfa 2024-11-18T03:17:23,719 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/305c4eedd62e41cea6fd93c964939dfa, entries=150, sequenceid=146, filesize=11.9 K 2024-11-18T03:17:23,728 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~131.15 KB/134300, heapSize ~373.34 KB/382304, currentSize=53.67 KB/54960 for 66ab176d63aa2fde549e454fab5778b4 in 1128ms, sequenceid=146, compaction requested=false 2024-11-18T03:17:23,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:23,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:23,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-18T03:17:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-18T03:17:23,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-18T03:17:23,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-11-18T03:17:23,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0680 sec 2024-11-18T03:17:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:23,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:17:23,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:23,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:23,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:23,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:23,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:23,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:23,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182067010a7eac4fa99ae86d245585e078_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899843449/Put/seqid=0 2024-11-18T03:17:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T03:17:23,831 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:23,831 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-18T03:17:23,844 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T03:17:23,846 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:23,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:23,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742437_1613 (size=14794) 2024-11-18T03:17:23,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899903861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899903863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899903863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899903864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899903869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T03:17:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899903985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899903985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899903985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899903985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899903985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T03:17:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899904201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899904202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899904202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899904204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899904204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:24,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182067010a7eac4fa99ae86d245585e078_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182067010a7eac4fa99ae86d245585e078_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:24,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/4538b9e12047475f83597c3cfe7452f1, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:24,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/4538b9e12047475f83597c3cfe7452f1 is 175, key is test_row_0/A:col10/1731899843449/Put/seqid=0 2024-11-18T03:17:24,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742438_1614 (size=39749) 2024-11-18T03:17:24,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T03:17:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899904513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899904513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899904513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899904513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899904516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:24,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:24,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/4538b9e12047475f83597c3cfe7452f1 2024-11-18T03:17:24,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/d46dae9f54ea4dc191f6cb189678c4a7 is 50, key is test_row_0/B:col10/1731899843449/Put/seqid=0 2024-11-18T03:17:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742439_1615 (size=12151) 2024-11-18T03:17:24,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:24,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:24,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:24,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T03:17:25,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899905023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899905033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:25,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899905033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899905034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899905035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:25,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:25,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/d46dae9f54ea4dc191f6cb189678c4a7 2024-11-18T03:17:25,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/6914334ebe014c139b6edb17f60060b5 is 50, key is test_row_0/C:col10/1731899843449/Put/seqid=0 2024-11-18T03:17:25,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742440_1616 (size=12151) 2024-11-18T03:17:25,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/6914334ebe014c139b6edb17f60060b5 2024-11-18T03:17:25,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:25,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:25,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:25,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:25,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/4538b9e12047475f83597c3cfe7452f1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/4538b9e12047475f83597c3cfe7452f1 2024-11-18T03:17:25,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/4538b9e12047475f83597c3cfe7452f1, entries=200, sequenceid=163, filesize=38.8 K 2024-11-18T03:17:25,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/d46dae9f54ea4dc191f6cb189678c4a7 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d46dae9f54ea4dc191f6cb189678c4a7 2024-11-18T03:17:25,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d46dae9f54ea4dc191f6cb189678c4a7, entries=150, sequenceid=163, filesize=11.9 K 2024-11-18T03:17:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/6914334ebe014c139b6edb17f60060b5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6914334ebe014c139b6edb17f60060b5 2024-11-18T03:17:25,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6914334ebe014c139b6edb17f60060b5, entries=150, sequenceid=163, filesize=11.9 K 2024-11-18T03:17:25,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 66ab176d63aa2fde549e454fab5778b4 in 1575ms, sequenceid=163, compaction requested=true 2024-11-18T03:17:25,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:25,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:25,376 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:25,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:25,377 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:25,378 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102122 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:25,378 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:25,378 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,378 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/acb7ee0c4bc8452fbb32ed482ba10b36, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ce133a22603c44828cf8e4a241594962, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/4538b9e12047475f83597c3cfe7452f1] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=99.7 K 2024-11-18T03:17:25,378 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,378 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/acb7ee0c4bc8452fbb32ed482ba10b36, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ce133a22603c44828cf8e4a241594962, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/4538b9e12047475f83597c3cfe7452f1] 2024-11-18T03:17:25,380 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting acb7ee0c4bc8452fbb32ed482ba10b36, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731899840036 2024-11-18T03:17:25,380 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:25,380 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:25,381 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,381 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a099cb973ed84b069490e96741f13784, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/67b5613f193e471b8ea38b39b6ba4c3a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d46dae9f54ea4dc191f6cb189678c4a7] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=35.7 K 2024-11-18T03:17:25,382 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a099cb973ed84b069490e96741f13784, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731899840036 2024-11-18T03:17:25,383 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce133a22603c44828cf8e4a241594962, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731899841271 2024-11-18T03:17:25,384 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 67b5613f193e471b8ea38b39b6ba4c3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731899841271 2024-11-18T03:17:25,388 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4538b9e12047475f83597c3cfe7452f1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899843449 2024-11-18T03:17:25,388 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d46dae9f54ea4dc191f6cb189678c4a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899843449 2024-11-18T03:17:25,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T03:17:25,446 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T03:17:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:25,458 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#523 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:25,459 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/82359a3cc5784cf7a2e09e80240a99d9 is 50, key is test_row_0/B:col10/1731899843449/Put/seqid=0 2024-11-18T03:17:25,465 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118a2caf7c4a1914c97a8935da887a74427_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:25,467 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118a2caf7c4a1914c97a8935da887a74427_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:25,467 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a2caf7c4a1914c97a8935da887a74427_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189834cc51ee1d4dbea7bc8ba8788613e7_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899843861/Put/seqid=0 2024-11-18T03:17:25,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742441_1617 (size=12459) 2024-11-18T03:17:25,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742442_1618 (size=4469) 2024-11-18T03:17:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742443_1619 (size=12304) 2024-11-18T03:17:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:25,605 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189834cc51ee1d4dbea7bc8ba8788613e7_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189834cc51ee1d4dbea7bc8ba8788613e7_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:25,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/b26ed93145d94fcba7fd0176ba853523, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/b26ed93145d94fcba7fd0176ba853523 is 175, key is test_row_0/A:col10/1731899843861/Put/seqid=0 2024-11-18T03:17:25,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742444_1620 (size=31105) 2024-11-18T03:17:25,930 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/82359a3cc5784cf7a2e09e80240a99d9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/82359a3cc5784cf7a2e09e80240a99d9 2024-11-18T03:17:25,940 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#522 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:25,941 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/7cc8973734e5432c87902eabdd131aa1 is 175, key is test_row_0/A:col10/1731899843449/Put/seqid=0 2024-11-18T03:17:25,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742445_1621 (size=31413) 2024-11-18T03:17:25,969 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into 82359a3cc5784cf7a2e09e80240a99d9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:25,969 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:25,969 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=13, startTime=1731899845377; duration=0sec 2024-11-18T03:17:25,969 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:25,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:25,970 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:25,984 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:25,984 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:25,984 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:25,984 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/d0bdbe4e938749668e236c492b14fd90, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/305c4eedd62e41cea6fd93c964939dfa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6914334ebe014c139b6edb17f60060b5] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=35.7 K 2024-11-18T03:17:25,996 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting d0bdbe4e938749668e236c492b14fd90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731899840036 2024-11-18T03:17:26,009 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 305c4eedd62e41cea6fd93c964939dfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731899841271 2024-11-18T03:17:26,013 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6914334ebe014c139b6edb17f60060b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899843449 2024-11-18T03:17:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T03:17:26,026 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/7cc8973734e5432c87902eabdd131aa1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/7cc8973734e5432c87902eabdd131aa1 2024-11-18T03:17:26,050 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:26,050 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/ef9067637c7b41388437bf91685b9f7d is 50, key is test_row_0/C:col10/1731899843449/Put/seqid=0 2024-11-18T03:17:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:26,057 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into 7cc8973734e5432c87902eabdd131aa1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:26,057 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:26,057 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=13, startTime=1731899845376; duration=0sec 2024-11-18T03:17:26,057 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:26,057 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:26,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/b26ed93145d94fcba7fd0176ba853523 2024-11-18T03:17:26,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742446_1622 (size=12459) 2024-11-18T03:17:26,095 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/ef9067637c7b41388437bf91685b9f7d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ef9067637c7b41388437bf91685b9f7d 2024-11-18T03:17:26,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/6bf2bbfba66e479bb0a3321397facdce is 50, key is test_row_0/B:col10/1731899843861/Put/seqid=0 2024-11-18T03:17:26,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742447_1623 (size=12151) 2024-11-18T03:17:26,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/6bf2bbfba66e479bb0a3321397facdce 2024-11-18T03:17:26,146 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into ef9067637c7b41388437bf91685b9f7d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:26,146 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:26,146 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=13, startTime=1731899845378; duration=0sec 2024-11-18T03:17:26,147 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:26,147 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899906127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899906132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899906149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899906149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899906149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e9e4195e5a304f37b5c85e6c2bf63e3a is 50, key is test_row_0/C:col10/1731899843861/Put/seqid=0 2024-11-18T03:17:26,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742448_1624 (size=12151) 2024-11-18T03:17:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899906253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899906268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899906271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899906272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899906272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899906471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899906481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899906483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899906491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899906491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,615 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e9e4195e5a304f37b5c85e6c2bf63e3a 2024-11-18T03:17:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/b26ed93145d94fcba7fd0176ba853523 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/b26ed93145d94fcba7fd0176ba853523 2024-11-18T03:17:26,629 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/b26ed93145d94fcba7fd0176ba853523, entries=150, sequenceid=182, filesize=30.4 K 2024-11-18T03:17:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/6bf2bbfba66e479bb0a3321397facdce as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6bf2bbfba66e479bb0a3321397facdce 2024-11-18T03:17:26,635 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6bf2bbfba66e479bb0a3321397facdce, entries=150, sequenceid=182, filesize=11.9 K 2024-11-18T03:17:26,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e9e4195e5a304f37b5c85e6c2bf63e3a as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e9e4195e5a304f37b5c85e6c2bf63e3a 2024-11-18T03:17:26,646 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e9e4195e5a304f37b5c85e6c2bf63e3a, entries=150, sequenceid=182, filesize=11.9 K 2024-11-18T03:17:26,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 66ab176d63aa2fde549e454fab5778b4 in 1199ms, sequenceid=182, compaction requested=false 2024-11-18T03:17:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-18T03:17:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-18T03:17:26,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-18T03:17:26,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8060 sec 2024-11-18T03:17:26,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.8340 sec 2024-11-18T03:17:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:26,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T03:17:26,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:26,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:26,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:26,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:26,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:26,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:26,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118431464dbfc884a2cb01dbaab0126d710_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899846134/Put/seqid=0 2024-11-18T03:17:26,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742449_1625 (size=12304) 2024-11-18T03:17:26,825 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:26,833 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118431464dbfc884a2cb01dbaab0126d710_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118431464dbfc884a2cb01dbaab0126d710_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:26,834 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/fdbb6069e2d44408b1ca6ccd04d2b7d5, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:26,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/fdbb6069e2d44408b1ca6ccd04d2b7d5 is 175, key is test_row_0/A:col10/1731899846134/Put/seqid=0 2024-11-18T03:17:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899906828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899906829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899906830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899906830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899906834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742450_1626 (size=31105) 2024-11-18T03:17:26,854 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/fdbb6069e2d44408b1ca6ccd04d2b7d5 2024-11-18T03:17:26,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/3b5d834638804bfab7dfaaeeb155ff8e is 50, key is test_row_0/B:col10/1731899846134/Put/seqid=0 2024-11-18T03:17:26,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742451_1627 (size=12151) 2024-11-18T03:17:26,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899906941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899906941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899906941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899906941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899906944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899907151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899907152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899907152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899907152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899907152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/3b5d834638804bfab7dfaaeeb155ff8e 2024-11-18T03:17:27,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/553fa051979f4221aa0e8acde6c57418 is 50, key is test_row_0/C:col10/1731899846134/Put/seqid=0 2024-11-18T03:17:27,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742452_1628 (size=12151) 2024-11-18T03:17:27,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/553fa051979f4221aa0e8acde6c57418 2024-11-18T03:17:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/fdbb6069e2d44408b1ca6ccd04d2b7d5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/fdbb6069e2d44408b1ca6ccd04d2b7d5 2024-11-18T03:17:27,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/fdbb6069e2d44408b1ca6ccd04d2b7d5, entries=150, sequenceid=201, filesize=30.4 K 2024-11-18T03:17:27,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/3b5d834638804bfab7dfaaeeb155ff8e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/3b5d834638804bfab7dfaaeeb155ff8e 2024-11-18T03:17:27,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/3b5d834638804bfab7dfaaeeb155ff8e, entries=150, sequenceid=201, filesize=11.9 K 2024-11-18T03:17:27,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/553fa051979f4221aa0e8acde6c57418 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/553fa051979f4221aa0e8acde6c57418 2024-11-18T03:17:27,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/553fa051979f4221aa0e8acde6c57418, entries=150, sequenceid=201, filesize=11.9 K 2024-11-18T03:17:27,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 66ab176d63aa2fde549e454fab5778b4 in 615ms, sequenceid=201, compaction requested=true 2024-11-18T03:17:27,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:27,401 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:27,402 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:27,402 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:27,402 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:27,402 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/7cc8973734e5432c87902eabdd131aa1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/b26ed93145d94fcba7fd0176ba853523, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/fdbb6069e2d44408b1ca6ccd04d2b7d5] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=91.4 K 2024-11-18T03:17:27,402 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:27,402 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/7cc8973734e5432c87902eabdd131aa1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/b26ed93145d94fcba7fd0176ba853523, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/fdbb6069e2d44408b1ca6ccd04d2b7d5] 2024-11-18T03:17:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:27,402 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:27,403 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc8973734e5432c87902eabdd131aa1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899843449 2024-11-18T03:17:27,403 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting b26ed93145d94fcba7fd0176ba853523, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731899843861 2024-11-18T03:17:27,404 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdbb6069e2d44408b1ca6ccd04d2b7d5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731899846117 2024-11-18T03:17:27,404 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:27,404 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:27,404 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:27,404 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/82359a3cc5784cf7a2e09e80240a99d9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6bf2bbfba66e479bb0a3321397facdce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/3b5d834638804bfab7dfaaeeb155ff8e] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=35.9 K 2024-11-18T03:17:27,404 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 82359a3cc5784cf7a2e09e80240a99d9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899843449 2024-11-18T03:17:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:27,405 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf2bbfba66e479bb0a3321397facdce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731899843861 2024-11-18T03:17:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:27,405 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5d834638804bfab7dfaaeeb155ff8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731899846117 2024-11-18T03:17:27,411 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:27,418 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:27,419 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/fe2da8f71c844bebabdd9a0eec80d13c is 50, key is test_row_0/B:col10/1731899846134/Put/seqid=0 2024-11-18T03:17:27,421 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118527802f226d5404c841ef7f7e38c47fb_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:27,423 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118527802f226d5404c841ef7f7e38c47fb_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:27,423 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118527802f226d5404c841ef7f7e38c47fb_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:27,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742453_1629 (size=12561) 2024-11-18T03:17:27,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742454_1630 (size=4469) 2024-11-18T03:17:27,460 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#531 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:27,461 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ae0f0790f22f4485b037930d6e7a4867 is 175, key is test_row_0/A:col10/1731899846134/Put/seqid=0 2024-11-18T03:17:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:27,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T03:17:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:27,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742455_1631 (size=31515) 2024-11-18T03:17:27,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899907482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899907486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899907487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899907487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899907489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118291be27521574ff79ef7e3d805b52e22_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899847461/Put/seqid=0 2024-11-18T03:17:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742456_1632 (size=12304) 2024-11-18T03:17:27,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:27,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118291be27521574ff79ef7e3d805b52e22_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118291be27521574ff79ef7e3d805b52e22_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:27,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/e65842850c054cad9985faa7acd6fe6b, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:27,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/e65842850c054cad9985faa7acd6fe6b is 175, key is test_row_0/A:col10/1731899847461/Put/seqid=0 2024-11-18T03:17:27,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899907605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899907605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899907605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899907605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899907608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742457_1633 (size=31105) 2024-11-18T03:17:27,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899907811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899907821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899907821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899907821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899907824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:27,868 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/fe2da8f71c844bebabdd9a0eec80d13c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/fe2da8f71c844bebabdd9a0eec80d13c 2024-11-18T03:17:27,890 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into fe2da8f71c844bebabdd9a0eec80d13c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:27,890 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:27,890 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=13, startTime=1731899847402; duration=0sec 2024-11-18T03:17:27,890 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:27,890 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:27,890 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:27,901 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:27,901 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:27,901 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:27,901 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ef9067637c7b41388437bf91685b9f7d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e9e4195e5a304f37b5c85e6c2bf63e3a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/553fa051979f4221aa0e8acde6c57418] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=35.9 K 2024-11-18T03:17:27,907 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ef9067637c7b41388437bf91685b9f7d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731899843449 2024-11-18T03:17:27,911 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e9e4195e5a304f37b5c85e6c2bf63e3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731899843861 2024-11-18T03:17:27,923 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 553fa051979f4221aa0e8acde6c57418, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731899846117 2024-11-18T03:17:27,948 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ae0f0790f22f4485b037930d6e7a4867 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae0f0790f22f4485b037930d6e7a4867 2024-11-18T03:17:27,968 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into ae0f0790f22f4485b037930d6e7a4867(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:27,968 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:27,968 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=13, startTime=1731899847401; duration=0sec 2024-11-18T03:17:27,969 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:27,969 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:27,998 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:27,999 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/944615574edb40b6a8e4209a53d8a836 is 50, key is test_row_0/C:col10/1731899846134/Put/seqid=0 2024-11-18T03:17:28,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742458_1634 (size=12561) 2024-11-18T03:17:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T03:17:28,031 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:28,033 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-18T03:17:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T03:17:28,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/e65842850c054cad9985faa7acd6fe6b 2024-11-18T03:17:28,047 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:28,049 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:28,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:28,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/8a713ceec9e840468fe3427d192d8bcd is 50, key is test_row_0/B:col10/1731899847461/Put/seqid=0 2024-11-18T03:17:28,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742459_1635 (size=12151) 2024-11-18T03:17:28,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/8a713ceec9e840468fe3427d192d8bcd 2024-11-18T03:17:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899908121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899908131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899908131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899908131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T03:17:28,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899908144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/a649045ecb4f4f4dacee118f84716053 is 50, key is test_row_0/C:col10/1731899847461/Put/seqid=0 2024-11-18T03:17:28,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T03:17:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742460_1636 (size=12151) 2024-11-18T03:17:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T03:17:28,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T03:17:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,429 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/944615574edb40b6a8e4209a53d8a836 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/944615574edb40b6a8e4209a53d8a836 2024-11-18T03:17:28,437 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into 944615574edb40b6a8e4209a53d8a836(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:28,437 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:28,437 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=13, startTime=1731899847405; duration=0sec 2024-11-18T03:17:28,437 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:28,437 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:28,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T03:17:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/a649045ecb4f4f4dacee118f84716053 2024-11-18T03:17:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899908632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899908642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899908653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899908656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T03:17:28,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899908664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:28,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/e65842850c054cad9985faa7acd6fe6b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/e65842850c054cad9985faa7acd6fe6b 2024-11-18T03:17:28,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T03:17:28,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:28,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:28,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/e65842850c054cad9985faa7acd6fe6b, entries=150, sequenceid=220, filesize=30.4 K 2024-11-18T03:17:28,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/8a713ceec9e840468fe3427d192d8bcd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/8a713ceec9e840468fe3427d192d8bcd 2024-11-18T03:17:28,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/8a713ceec9e840468fe3427d192d8bcd, entries=150, sequenceid=220, filesize=11.9 K 2024-11-18T03:17:28,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/a649045ecb4f4f4dacee118f84716053 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/a649045ecb4f4f4dacee118f84716053 2024-11-18T03:17:28,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/a649045ecb4f4f4dacee118f84716053, entries=150, sequenceid=220, filesize=11.9 K 2024-11-18T03:17:28,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 66ab176d63aa2fde549e454fab5778b4 in 1259ms, sequenceid=220, compaction requested=false 2024-11-18T03:17:28,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:28,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T03:17:28,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:28,852 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:17:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118783c55f60a2d484abdd2a19db5bd63b8_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899847480/Put/seqid=0 2024-11-18T03:17:28,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742461_1637 (size=12304) 2024-11-18T03:17:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:28,921 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118783c55f60a2d484abdd2a19db5bd63b8_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118783c55f60a2d484abdd2a19db5bd63b8_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:28,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/dd8f891955164b788166f80b43072bb1, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/dd8f891955164b788166f80b43072bb1 is 175, key is test_row_0/A:col10/1731899847480/Put/seqid=0 2024-11-18T03:17:28,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742462_1638 (size=31105) 2024-11-18T03:17:28,941 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/dd8f891955164b788166f80b43072bb1 2024-11-18T03:17:28,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/c791105f61a24bb88e6aff13348d5b0e is 50, key is test_row_0/B:col10/1731899847480/Put/seqid=0 2024-11-18T03:17:28,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742463_1639 (size=12151) 2024-11-18T03:17:28,970 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/c791105f61a24bb88e6aff13348d5b0e 2024-11-18T03:17:28,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/910df35cefb647eb83b38f5ae742b529 is 50, key is test_row_0/C:col10/1731899847480/Put/seqid=0 2024-11-18T03:17:29,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742464_1640 (size=12151) 2024-11-18T03:17:29,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/910df35cefb647eb83b38f5ae742b529 2024-11-18T03:17:29,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/dd8f891955164b788166f80b43072bb1 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd8f891955164b788166f80b43072bb1 2024-11-18T03:17:29,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd8f891955164b788166f80b43072bb1, entries=150, sequenceid=236, filesize=30.4 K 2024-11-18T03:17:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/c791105f61a24bb88e6aff13348d5b0e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c791105f61a24bb88e6aff13348d5b0e 2024-11-18T03:17:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,051 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c791105f61a24bb88e6aff13348d5b0e, entries=150, sequenceid=236, filesize=11.9 K 2024-11-18T03:17:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/910df35cefb647eb83b38f5ae742b529 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/910df35cefb647eb83b38f5ae742b529 2024-11-18T03:17:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,056 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/910df35cefb647eb83b38f5ae742b529, entries=150, sequenceid=236, filesize=11.9 K 2024-11-18T03:17:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,057 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 66ab176d63aa2fde549e454fab5778b4 in 205ms, sequenceid=236, compaction requested=true 2024-11-18T03:17:29,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:29,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:29,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-18T03:17:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-18T03:17:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-18T03:17:29,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-11-18T03:17:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0400 sec 2024-11-18T03:17:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T03:17:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,180 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,184 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-18T03:17:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T03:17:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,198 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,204 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:29,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T03:17:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T03:17:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-18T03:17:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-18T03:17:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-18T03:17:29,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 173 msec 2024-11-18T03:17:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 198 msec 2024-11-18T03:17:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T03:17:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,520 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,524 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-18T03:17:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T03:17:29,534 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,535 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T03:17:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T03:17:29,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,697 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T03:17:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:29,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118605a4ef594d1470887a6a1fd016b3f44_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899849689/Put/seqid=0 2024-11-18T03:17:29,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899849734, max timestamp=1731899849749Num uniques 859; 2024-11-18T03:17:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:17:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899849734, max timestamp=1731899849749Num uniques 859; 2024-11-18T03:17:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:17:29,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899849734, max timestamp=1731899849749Num uniques 859; 2024-11-18T03:17:29,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:17:29,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742465_1641 (size=19774) 2024-11-18T03:17:29,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:29,762 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118605a4ef594d1470887a6a1fd016b3f44_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118605a4ef594d1470887a6a1fd016b3f44_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:29,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d41b1682de7f456abfa486639262c2c4, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:29,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d41b1682de7f456abfa486639262c2c4 is 175, key is test_row_0/A:col10/1731899849689/Put/seqid=0 2024-11-18T03:17:29,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742466_1642 (size=57033) 2024-11-18T03:17:29,785 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d41b1682de7f456abfa486639262c2c4 2024-11-18T03:17:29,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899909785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899909785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899909786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899909788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899909793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/567e71bd61814c11998188fe53b68a27 is 50, key is test_row_0/B:col10/1731899849689/Put/seqid=0 2024-11-18T03:17:29,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742467_1643 (size=12151) 2024-11-18T03:17:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T03:17:29,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899909904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899909905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899909912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899909913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:29,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899909913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:29,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:29,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899910111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899910122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899910122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899910122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899910122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T03:17:30,236 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/567e71bd61814c11998188fe53b68a27 2024-11-18T03:17:30,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e59bb64b653f4415b89d9d3c84767298 is 50, key is test_row_0/C:col10/1731899849689/Put/seqid=0 2024-11-18T03:17:30,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742468_1644 (size=12151) 2024-11-18T03:17:30,323 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e59bb64b653f4415b89d9d3c84767298 2024-11-18T03:17:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d41b1682de7f456abfa486639262c2c4 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d41b1682de7f456abfa486639262c2c4 2024-11-18T03:17:30,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899910430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,444 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d41b1682de7f456abfa486639262c2c4, entries=300, sequenceid=246, filesize=55.7 K 2024-11-18T03:17:30,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899910445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899910445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899910445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899910445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/567e71bd61814c11998188fe53b68a27 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/567e71bd61814c11998188fe53b68a27 2024-11-18T03:17:30,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/567e71bd61814c11998188fe53b68a27, entries=150, sequenceid=246, filesize=11.9 K 2024-11-18T03:17:30,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/e59bb64b653f4415b89d9d3c84767298 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e59bb64b653f4415b89d9d3c84767298 2024-11-18T03:17:30,584 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e59bb64b653f4415b89d9d3c84767298, entries=150, sequenceid=246, filesize=11.9 K 2024-11-18T03:17:30,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 66ab176d63aa2fde549e454fab5778b4 in 899ms, sequenceid=246, compaction requested=true 2024-11-18T03:17:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-18T03:17:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-18T03:17:30,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-18T03:17:30,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0700 sec 2024-11-18T03:17:30,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.0840 sec 2024-11-18T03:17:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T03:17:30,671 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:30,671 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-18T03:17:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T03:17:30,673 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:30,674 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:30,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T03:17:30,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T03:17:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:30,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-18T03:17:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:30,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:30,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:30,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e077f81fe1c74b5a823cd24c0010ea3e_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899849782/Put/seqid=0 2024-11-18T03:17:30,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742469_1645 (size=12454) 2024-11-18T03:17:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T03:17:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:31,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:31,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899911030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899911031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899911036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899911048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899911050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899911151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899911161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899911161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899911164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899911172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T03:17:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:31,341 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e077f81fe1c74b5a823cd24c0010ea3e_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e077f81fe1c74b5a823cd24c0010ea3e_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/0fe671c0dad44e7eaecb1ec905b14993, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/0fe671c0dad44e7eaecb1ec905b14993 is 175, key is test_row_0/A:col10/1731899849782/Put/seqid=0 2024-11-18T03:17:31,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899911365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899911372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899911372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899911372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742470_1646 (size=31255) 2024-11-18T03:17:31,392 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/0fe671c0dad44e7eaecb1ec905b14993 2024-11-18T03:17:31,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899911396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f1135e05982748d68ed8f936755e116d is 50, key is test_row_0/B:col10/1731899849782/Put/seqid=0 2024-11-18T03:17:31,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742471_1647 (size=12301) 2024-11-18T03:17:31,492 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f1135e05982748d68ed8f936755e116d 2024-11-18T03:17:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/38c66ec6e0e64530b12af5d34203a026 is 50, key is test_row_0/C:col10/1731899849782/Put/seqid=0 2024-11-18T03:17:31,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742472_1648 (size=12301) 2024-11-18T03:17:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899911682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899911691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899911692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899911696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899911716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T03:17:32,021 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/38c66ec6e0e64530b12af5d34203a026 2024-11-18T03:17:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/0fe671c0dad44e7eaecb1ec905b14993 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/0fe671c0dad44e7eaecb1ec905b14993 2024-11-18T03:17:32,049 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/0fe671c0dad44e7eaecb1ec905b14993, entries=150, sequenceid=282, filesize=30.5 K 2024-11-18T03:17:32,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/f1135e05982748d68ed8f936755e116d as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f1135e05982748d68ed8f936755e116d 2024-11-18T03:17:32,086 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f1135e05982748d68ed8f936755e116d, entries=150, sequenceid=282, filesize=12.0 K 2024-11-18T03:17:32,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/38c66ec6e0e64530b12af5d34203a026 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/38c66ec6e0e64530b12af5d34203a026 2024-11-18T03:17:32,136 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/38c66ec6e0e64530b12af5d34203a026, entries=150, sequenceid=282, filesize=12.0 K 2024-11-18T03:17:32,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 66ab176d63aa2fde549e454fab5778b4 in 1301ms, sequenceid=282, compaction requested=true 2024-11-18T03:17:32,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-18T03:17:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-18T03:17:32,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-18T03:17:32,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-11-18T03:17:32,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4890 sec 2024-11-18T03:17:32,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T03:17:32,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:32,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:32,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:32,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:32,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:32,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:32,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111829f0c7d0b0fe4d628deffbb2bb8d2bd5_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899852206/Put/seqid=0 2024-11-18T03:17:32,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742473_1649 (size=14994) 2024-11-18T03:17:32,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:32,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111829f0c7d0b0fe4d628deffbb2bb8d2bd5_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111829f0c7d0b0fe4d628deffbb2bb8d2bd5_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:32,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/bc16e46b165c4b229a7c7b8d1b7dda43, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:32,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/bc16e46b165c4b229a7c7b8d1b7dda43 is 175, key is test_row_0/A:col10/1731899852206/Put/seqid=0 2024-11-18T03:17:32,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731899852269, max timestamp=1731899852295Num uniques 859; 2024-11-18T03:17:32,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T03:17:32,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899852271, max timestamp=1731899852296Num uniques 859; 2024-11-18T03:17:32,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:32,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899852271, max timestamp=1731899852296Num uniques 859; 2024-11-18T03:17:32,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:32,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742474_1650 (size=39949) 2024-11-18T03:17:32,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899912356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899912360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899912378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899912381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899912396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899912491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899912491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899912501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899912501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899912521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899912713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899912714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899912724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899912728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/bc16e46b165c4b229a7c7b8d1b7dda43 2024-11-18T03:17:32,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899912733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:32,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/108219d8e3254c38af38df6dd58bffac is 50, key is test_row_0/B:col10/1731899852206/Put/seqid=0 2024-11-18T03:17:32,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742475_1651 (size=12301) 2024-11-18T03:17:32,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/108219d8e3254c38af38df6dd58bffac 2024-11-18T03:17:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T03:17:32,845 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:32,846 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-18T03:17:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-18T03:17:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:32,853 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T03:17:32,854 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T03:17:32,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T03:17:32,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/3d8c40ca689444cea1dd057577c9f785 is 50, key is test_row_0/C:col10/1731899852206/Put/seqid=0 2024-11-18T03:17:32,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742476_1652 (size=12301) 2024-11-18T03:17:32,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/3d8c40ca689444cea1dd057577c9f785 2024-11-18T03:17:32,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/bc16e46b165c4b229a7c7b8d1b7dda43 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/bc16e46b165c4b229a7c7b8d1b7dda43 2024-11-18T03:17:32,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/bc16e46b165c4b229a7c7b8d1b7dda43, entries=200, sequenceid=293, filesize=39.0 K 2024-11-18T03:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/108219d8e3254c38af38df6dd58bffac as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/108219d8e3254c38af38df6dd58bffac 2024-11-18T03:17:32,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/108219d8e3254c38af38df6dd58bffac, entries=150, sequenceid=293, filesize=12.0 K 2024-11-18T03:17:32,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/3d8c40ca689444cea1dd057577c9f785 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3d8c40ca689444cea1dd057577c9f785 2024-11-18T03:17:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:33,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3d8c40ca689444cea1dd057577c9f785, entries=150, sequenceid=293, filesize=12.0 K 2024-11-18T03:17:33,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 66ab176d63aa2fde549e454fab5778b4 in 799ms, sequenceid=293, compaction requested=true 2024-11-18T03:17:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:17:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T03:17:33,010 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-18T03:17:33,016 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-18T03:17:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,028 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 221962 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-18T03:17:33,028 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:33,028 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,028 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae0f0790f22f4485b037930d6e7a4867, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/e65842850c054cad9985faa7acd6fe6b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd8f891955164b788166f80b43072bb1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d41b1682de7f456abfa486639262c2c4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/0fe671c0dad44e7eaecb1ec905b14993, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/bc16e46b165c4b229a7c7b8d1b7dda43] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=216.8 K 2024-11-18T03:17:33,028 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,028 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae0f0790f22f4485b037930d6e7a4867, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/e65842850c054cad9985faa7acd6fe6b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd8f891955164b788166f80b43072bb1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d41b1682de7f456abfa486639262c2c4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/0fe671c0dad44e7eaecb1ec905b14993, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/bc16e46b165c4b229a7c7b8d1b7dda43] 2024-11-18T03:17:33,029 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0f0790f22f4485b037930d6e7a4867, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731899846117 2024-11-18T03:17:33,031 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65842850c054cad9985faa7acd6fe6b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731899846829 2024-11-18T03:17:33,033 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8f891955164b788166f80b43072bb1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731899847480 2024-11-18T03:17:33,033 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73616 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-18T03:17:33,034 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41b1682de7f456abfa486639262c2c4, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731899849687 2024-11-18T03:17:33,034 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:33,034 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,034 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/944615574edb40b6a8e4209a53d8a836, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/a649045ecb4f4f4dacee118f84716053, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/910df35cefb647eb83b38f5ae742b529, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e59bb64b653f4415b89d9d3c84767298, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/38c66ec6e0e64530b12af5d34203a026, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3d8c40ca689444cea1dd057577c9f785] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=71.9 K 2024-11-18T03:17:33,035 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe671c0dad44e7eaecb1ec905b14993, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731899849782 2024-11-18T03:17:33,035 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 944615574edb40b6a8e4209a53d8a836, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731899846117 2024-11-18T03:17:33,035 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc16e46b165c4b229a7c7b8d1b7dda43, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899851029 2024-11-18T03:17:33,036 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting a649045ecb4f4f4dacee118f84716053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731899846829 2024-11-18T03:17:33,036 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 910df35cefb647eb83b38f5ae742b529, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731899847480 2024-11-18T03:17:33,037 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting e59bb64b653f4415b89d9d3c84767298, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731899849687 2024-11-18T03:17:33,037 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c66ec6e0e64530b12af5d34203a026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731899849782 2024-11-18T03:17:33,037 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d8c40ca689444cea1dd057577c9f785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899851029 2024-11-18T03:17:33,058 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:33,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-18T03:17:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:33,069 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#550 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:33,069 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/41cbcfd4c2524f17b0382018cb55bcdc is 50, key is test_row_0/C:col10/1731899852206/Put/seqid=0 2024-11-18T03:17:33,077 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411183bbb2b4387b240c5950e023f214335c3_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:33,083 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411183bbb2b4387b240c5950e023f214335c3_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:33,083 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183bbb2b4387b240c5950e023f214335c3_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:33,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899913089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899913097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899913109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899913116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899913121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111828dd8a373f924432852838f0f6d66183_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899852373/Put/seqid=0 2024-11-18T03:17:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:33,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742477_1653 (size=12915) 2024-11-18T03:17:33,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:33,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:33,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742478_1654 (size=4469) 2024-11-18T03:17:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899913221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899913232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899913244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742479_1655 (size=14994) 2024-11-18T03:17:33,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899913251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:33,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899913265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,303 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111828dd8a373f924432852838f0f6d66183_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828dd8a373f924432852838f0f6d66183_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:33,309 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2fab54b920ec4c29a7e58c377526a6f0, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:33,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2fab54b920ec4c29a7e58c377526a6f0 is 175, key is test_row_0/A:col10/1731899852373/Put/seqid=0 2024-11-18T03:17:33,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742480_1656 (size=39949) 2024-11-18T03:17:33,379 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2fab54b920ec4c29a7e58c377526a6f0 2024-11-18T03:17:33,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/30a0bcca26374fa58d88ee358309be10 is 50, key is test_row_0/B:col10/1731899852373/Put/seqid=0 2024-11-18T03:17:33,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899913452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899913452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742481_1657 (size=12301) 2024-11-18T03:17:33,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899913461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/30a0bcca26374fa58d88ee358309be10 2024-11-18T03:17:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:33,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899913482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899913483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/0dccef52e85943caa06a1e53c774ae15 is 50, key is test_row_0/C:col10/1731899852373/Put/seqid=0 2024-11-18T03:17:33,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742482_1658 (size=12301) 2024-11-18T03:17:33,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/0dccef52e85943caa06a1e53c774ae15 2024-11-18T03:17:33,609 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/41cbcfd4c2524f17b0382018cb55bcdc as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/41cbcfd4c2524f17b0382018cb55bcdc 2024-11-18T03:17:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2fab54b920ec4c29a7e58c377526a6f0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2fab54b920ec4c29a7e58c377526a6f0 2024-11-18T03:17:33,638 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#549 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:33,638 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d1bd596e028f481b8fc4b3950a7f6e43 is 175, key is test_row_0/A:col10/1731899852206/Put/seqid=0 2024-11-18T03:17:33,647 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into 41cbcfd4c2524f17b0382018cb55bcdc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:33,647 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:33,647 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=10, startTime=1731899853010; duration=0sec 2024-11-18T03:17:33,648 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:33,648 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:33,648 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-18T03:17:33,651 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73616 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-18T03:17:33,651 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:33,651 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,652 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/fe2da8f71c844bebabdd9a0eec80d13c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/8a713ceec9e840468fe3427d192d8bcd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c791105f61a24bb88e6aff13348d5b0e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/567e71bd61814c11998188fe53b68a27, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f1135e05982748d68ed8f936755e116d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/108219d8e3254c38af38df6dd58bffac] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=71.9 K 2024-11-18T03:17:33,652 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2da8f71c844bebabdd9a0eec80d13c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731899846117 2024-11-18T03:17:33,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2fab54b920ec4c29a7e58c377526a6f0, entries=200, sequenceid=330, filesize=39.0 K 2024-11-18T03:17:33,653 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a713ceec9e840468fe3427d192d8bcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731899846829 2024-11-18T03:17:33,653 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c791105f61a24bb88e6aff13348d5b0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731899847480 2024-11-18T03:17:33,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/30a0bcca26374fa58d88ee358309be10 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/30a0bcca26374fa58d88ee358309be10 2024-11-18T03:17:33,654 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 567e71bd61814c11998188fe53b68a27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731899849687 2024-11-18T03:17:33,654 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f1135e05982748d68ed8f936755e116d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731899849782 2024-11-18T03:17:33,655 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 108219d8e3254c38af38df6dd58bffac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899851029 2024-11-18T03:17:33,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742483_1659 (size=31869) 2024-11-18T03:17:33,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:33,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:33,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/30a0bcca26374fa58d88ee358309be10, entries=150, sequenceid=330, filesize=12.0 K 2024-11-18T03:17:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/0dccef52e85943caa06a1e53c774ae15 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/0dccef52e85943caa06a1e53c774ae15 2024-11-18T03:17:33,700 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/d1bd596e028f481b8fc4b3950a7f6e43 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d1bd596e028f481b8fc4b3950a7f6e43 2024-11-18T03:17:33,705 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into d1bd596e028f481b8fc4b3950a7f6e43(size=31.1 K), total size for store is 70.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:33,705 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:33,705 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=10, startTime=1731899853010; duration=0sec 2024-11-18T03:17:33,705 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:33,705 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:33,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/0dccef52e85943caa06a1e53c774ae15, entries=150, sequenceid=330, filesize=12.0 K 2024-11-18T03:17:33,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for 66ab176d63aa2fde549e454fab5778b4 in 642ms, sequenceid=330, compaction requested=false 2024-11-18T03:17:33,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:33,718 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#554 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:33,719 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/331a6f31afb34dbc872edea1cc1fbddd is 50, key is test_row_0/B:col10/1731899852206/Put/seqid=0 2024-11-18T03:17:33,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742484_1660 (size=12915) 2024-11-18T03:17:33,773 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/331a6f31afb34dbc872edea1cc1fbddd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/331a6f31afb34dbc872edea1cc1fbddd 2024-11-18T03:17:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:33,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T03:17:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:33,818 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into 331a6f31afb34dbc872edea1cc1fbddd(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:33,818 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:33,818 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=10, startTime=1731899853010; duration=0sec 2024-11-18T03:17:33,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111859d1fee34d9744f99f97b42b6059c12a_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899853101/Put/seqid=0 2024-11-18T03:17:33,818 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:33,818 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:33,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:33,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:33,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:33,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:33,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742485_1661 (size=12454) 2024-11-18T03:17:33,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899913839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899853825, max timestamp=1731899853847Num uniques 859; 2024-11-18T03:17:33,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:33,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899853825, max timestamp=1731899853847Num uniques 859; 2024-11-18T03:17:33,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899853825, max timestamp=1731899853847Num uniques 859; 2024-11-18T03:17:33,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899913888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899913889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899913891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899913892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899913964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:33,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:34,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:34,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899914004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899914004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899914011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899914011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899914181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899914211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899914222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899914222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899914222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:34,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111859d1fee34d9744f99f97b42b6059c12a_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111859d1fee34d9744f99f97b42b6059c12a_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:34,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/a4bb6ad141644be1855f5949877f37d0, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:34,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/a4bb6ad141644be1855f5949877f37d0 is 175, key is test_row_0/A:col10/1731899853101/Put/seqid=0 2024-11-18T03:17:34,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742486_1662 (size=31255) 2024-11-18T03:17:34,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:34,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899914491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899914523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899914531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899914541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899914541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:34,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/a4bb6ad141644be1855f5949877f37d0 2024-11-18T03:17:34,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/c3d70a3ed015456582b57c1048abc1e2 is 50, key is test_row_0/B:col10/1731899853101/Put/seqid=0 2024-11-18T03:17:34,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:34,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:34,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742487_1663 (size=12301) 2024-11-18T03:17:34,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:34,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:35,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899915001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:35,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899915033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:35,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899915054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899915062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899915062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:35,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:35,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/c3d70a3ed015456582b57c1048abc1e2 2024-11-18T03:17:35,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/c1a20e3d48bc41da9012983eae38b38e is 50, key is test_row_0/C:col10/1731899853101/Put/seqid=0 2024-11-18T03:17:35,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742488_1664 (size=12301) 2024-11-18T03:17:35,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/c1a20e3d48bc41da9012983eae38b38e 2024-11-18T03:17:35,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:35,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:35,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:35,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:17:35,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/a4bb6ad141644be1855f5949877f37d0 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/a4bb6ad141644be1855f5949877f37d0 2024-11-18T03:17:35,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/a4bb6ad141644be1855f5949877f37d0, entries=150, sequenceid=343, filesize=30.5 K 2024-11-18T03:17:35,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/c3d70a3ed015456582b57c1048abc1e2 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c3d70a3ed015456582b57c1048abc1e2 2024-11-18T03:17:35,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c3d70a3ed015456582b57c1048abc1e2, entries=150, sequenceid=343, filesize=12.0 K 2024-11-18T03:17:35,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/c1a20e3d48bc41da9012983eae38b38e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/c1a20e3d48bc41da9012983eae38b38e 2024-11-18T03:17:35,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/c1a20e3d48bc41da9012983eae38b38e, entries=150, sequenceid=343, filesize=12.0 K 2024-11-18T03:17:35,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 66ab176d63aa2fde549e454fab5778b4 in 1557ms, sequenceid=343, compaction requested=true 2024-11-18T03:17:35,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:35,341 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:17:35,343 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:35,351 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:35,351 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:35,351 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,351 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d1bd596e028f481b8fc4b3950a7f6e43, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2fab54b920ec4c29a7e58c377526a6f0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/a4bb6ad141644be1855f5949877f37d0] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=100.7 K 2024-11-18T03:17:35,351 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,351 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d1bd596e028f481b8fc4b3950a7f6e43, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2fab54b920ec4c29a7e58c377526a6f0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/a4bb6ad141644be1855f5949877f37d0] 2024-11-18T03:17:35,352 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1bd596e028f481b8fc4b3950a7f6e43, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899851029 2024-11-18T03:17:35,352 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fab54b920ec4c29a7e58c377526a6f0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731899852352 2024-11-18T03:17:35,355 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4bb6ad141644be1855f5949877f37d0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731899853101 2024-11-18T03:17:35,358 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:35,358 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:35,358 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,358 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/331a6f31afb34dbc872edea1cc1fbddd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/30a0bcca26374fa58d88ee358309be10, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c3d70a3ed015456582b57c1048abc1e2] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=36.6 K 2024-11-18T03:17:35,363 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 331a6f31afb34dbc872edea1cc1fbddd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899851029 2024-11-18T03:17:35,367 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 30a0bcca26374fa58d88ee358309be10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731899852352 2024-11-18T03:17:35,372 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d70a3ed015456582b57c1048abc1e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731899853101 2024-11-18T03:17:35,395 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:35,401 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411186242daaf31424613a0bc0792010af4c7_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:35,403 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411186242daaf31424613a0bc0792010af4c7_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:35,403 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186242daaf31424613a0bc0792010af4c7_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:35,438 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:35,439 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/5e225a218c1c4d50901f554c3f5f4007 is 50, key is test_row_0/B:col10/1731899853101/Put/seqid=0 2024-11-18T03:17:35,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T03:17:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-18T03:17:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742489_1665 (size=4469) 2024-11-18T03:17:35,457 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#558 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:35,458 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/c33dd5215f02448c8ac1e96b36bf730e is 175, key is test_row_0/A:col10/1731899853101/Put/seqid=0 2024-11-18T03:17:35,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742490_1666 (size=13017) 2024-11-18T03:17:35,511 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/5e225a218c1c4d50901f554c3f5f4007 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5e225a218c1c4d50901f554c3f5f4007 2024-11-18T03:17:35,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742491_1667 (size=31971) 2024-11-18T03:17:35,518 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into 5e225a218c1c4d50901f554c3f5f4007(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:35,518 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:35,518 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=13, startTime=1731899855341; duration=0sec 2024-11-18T03:17:35,518 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:35,518 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:35,518 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:35,520 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:35,520 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:35,520 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:35,520 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/41cbcfd4c2524f17b0382018cb55bcdc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/0dccef52e85943caa06a1e53c774ae15, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/c1a20e3d48bc41da9012983eae38b38e] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=36.6 K 2024-11-18T03:17:35,521 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 41cbcfd4c2524f17b0382018cb55bcdc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731899851029 2024-11-18T03:17:35,521 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dccef52e85943caa06a1e53c774ae15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731899852352 2024-11-18T03:17:35,522 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a20e3d48bc41da9012983eae38b38e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731899853101 2024-11-18T03:17:35,523 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/c33dd5215f02448c8ac1e96b36bf730e as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c33dd5215f02448c8ac1e96b36bf730e 2024-11-18T03:17:35,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187a63b79024814eb3801810d992bc7f49_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899853886/Put/seqid=0 2024-11-18T03:17:35,536 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into c33dd5215f02448c8ac1e96b36bf730e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:35,536 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:35,537 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=13, startTime=1731899855340; duration=0sec 2024-11-18T03:17:35,537 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:35,537 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:35,540 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#561 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:35,540 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/3932e3d622fc4ee0a4f9d9094244a246 is 50, key is test_row_0/C:col10/1731899853101/Put/seqid=0 2024-11-18T03:17:35,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742492_1668 (size=12454) 2024-11-18T03:17:35,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742493_1669 (size=13017) 2024-11-18T03:17:35,601 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/3932e3d622fc4ee0a4f9d9094244a246 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3932e3d622fc4ee0a4f9d9094244a246 2024-11-18T03:17:35,605 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into 3932e3d622fc4ee0a4f9d9094244a246(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:35,605 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:35,605 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=13, startTime=1731899855341; duration=0sec 2024-11-18T03:17:35,605 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:35,605 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:35,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:36,016 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187a63b79024814eb3801810d992bc7f49_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187a63b79024814eb3801810d992bc7f49_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/423eee99fd4a4c968b66a542a6b2faa6, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:36,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/423eee99fd4a4c968b66a542a6b2faa6 is 175, key is test_row_0/A:col10/1731899853886/Put/seqid=0 2024-11-18T03:17:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. as already flushing 2024-11-18T03:17:36,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742494_1670 (size=31255) 2024-11-18T03:17:36,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/423eee99fd4a4c968b66a542a6b2faa6 2024-11-18T03:17:36,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899916066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899916068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899916072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,077 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_2' in 66ab176d63aa2fde549e454fab5778b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:17:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899916081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,086 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 66ab176d63aa2fde549e454fab5778b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:17:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899916085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,091 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fd862c3a04f9,38549,1731899684479 for 'test_row_0' in 66ab176d63aa2fde549e454fab5778b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T03:17:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/313f8cf8a5f04709abc1b76af2b08dec is 50, key is test_row_0/B:col10/1731899853886/Put/seqid=0 2024-11-18T03:17:36,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742495_1671 (size=12301) 2024-11-18T03:17:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899916184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899916186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899916403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899916412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,541 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/313f8cf8a5f04709abc1b76af2b08dec 2024-11-18T03:17:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/f8ce442960d540ff9d9059a4fec64a9c is 50, key is test_row_0/C:col10/1731899853886/Put/seqid=0 2024-11-18T03:17:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742496_1672 (size=12301) 2024-11-18T03:17:36,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899916721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899916721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:37,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/f8ce442960d540ff9d9059a4fec64a9c 2024-11-18T03:17:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/423eee99fd4a4c968b66a542a6b2faa6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/423eee99fd4a4c968b66a542a6b2faa6 2024-11-18T03:17:37,103 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/423eee99fd4a4c968b66a542a6b2faa6, entries=150, sequenceid=380, filesize=30.5 K 2024-11-18T03:17:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/313f8cf8a5f04709abc1b76af2b08dec as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/313f8cf8a5f04709abc1b76af2b08dec 2024-11-18T03:17:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,122 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/313f8cf8a5f04709abc1b76af2b08dec, entries=150, sequenceid=380, filesize=12.0 K 2024-11-18T03:17:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/f8ce442960d540ff9d9059a4fec64a9c as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/f8ce442960d540ff9d9059a4fec64a9c 2024-11-18T03:17:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/f8ce442960d540ff9d9059a4fec64a9c, entries=150, sequenceid=380, filesize=12.0 K 2024-11-18T03:17:37,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 66ab176d63aa2fde549e454fab5778b4 in 1708ms, sequenceid=380, compaction requested=false 2024-11-18T03:17:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fd862c3a04f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-18T03:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-18T03:17:37,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-18T03:17:37,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3040 sec 2024-11-18T03:17:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 4.3160 sec 2024-11-18T03:17:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:37,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T03:17:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182c8b4a4294c54f9e9570d5f550798065_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899856057/Put/seqid=0 2024-11-18T03:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742497_1673 (size=12454) 2024-11-18T03:17:37,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182c8b4a4294c54f9e9570d5f550798065_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182c8b4a4294c54f9e9570d5f550798065_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2de6f911a4964e099584c4c3f18d3eaa, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2de6f911a4964e099584c4c3f18d3eaa is 175, key is test_row_0/A:col10/1731899856057/Put/seqid=0 2024-11-18T03:17:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:37,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742498_1674 (size=31251) 2024-11-18T03:17:37,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2de6f911a4964e099584c4c3f18d3eaa 2024-11-18T03:17:37,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731899857326, max timestamp=1731899857354Num uniques 859; 2024-11-18T03:17:37,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T03:17:37,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/70890188c28a4ba8ae41b4de62160993 is 50, key is test_row_0/B:col10/1731899856057/Put/seqid=0 2024-11-18T03:17:37,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899857326, max timestamp=1731899857354Num uniques 859; 2024-11-18T03:17:37,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:37,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731899857326, max timestamp=1731899857354Num uniques 859; 2024-11-18T03:17:37,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fd862c3a04f9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T03:17:37,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742499_1675 (size=9857) 2024-11-18T03:17:37,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899917436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899917440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:37,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899917551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:37,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899917552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:37,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899917771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:37,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899917772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:37,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/70890188c28a4ba8ae41b4de62160993 2024-11-18T03:17:37,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/ed43438af7de4dd393f1224fd9956a84 is 50, key is test_row_0/C:col10/1731899856057/Put/seqid=0 2024-11-18T03:17:37,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742500_1676 (size=9857) 2024-11-18T03:17:37,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/ed43438af7de4dd393f1224fd9956a84 2024-11-18T03:17:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/2de6f911a4964e099584c4c3f18d3eaa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2de6f911a4964e099584c4c3f18d3eaa 2024-11-18T03:17:37,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2de6f911a4964e099584c4c3f18d3eaa, entries=150, sequenceid=395, filesize=30.5 K 2024-11-18T03:17:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/70890188c28a4ba8ae41b4de62160993 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/70890188c28a4ba8ae41b4de62160993 2024-11-18T03:17:38,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/70890188c28a4ba8ae41b4de62160993, entries=100, sequenceid=395, filesize=9.6 K 2024-11-18T03:17:38,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/ed43438af7de4dd393f1224fd9956a84 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ed43438af7de4dd393f1224fd9956a84 2024-11-18T03:17:38,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ed43438af7de4dd393f1224fd9956a84, entries=100, sequenceid=395, filesize=9.6 K 2024-11-18T03:17:38,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for 66ab176d63aa2fde549e454fab5778b4 in 818ms, sequenceid=395, compaction requested=true 2024-11-18T03:17:38,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:38,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:38,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:38,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:38,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T03:17:38,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:38,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T03:17:38,063 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:38,063 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:38,070 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94477 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:38,070 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:38,071 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:38,071 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c33dd5215f02448c8ac1e96b36bf730e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/423eee99fd4a4c968b66a542a6b2faa6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2de6f911a4964e099584c4c3f18d3eaa] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=92.3 K 2024-11-18T03:17:38,071 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:38,071 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c33dd5215f02448c8ac1e96b36bf730e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/423eee99fd4a4c968b66a542a6b2faa6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2de6f911a4964e099584c4c3f18d3eaa] 2024-11-18T03:17:38,074 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting c33dd5215f02448c8ac1e96b36bf730e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731899853101 2024-11-18T03:17:38,074 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:38,074 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:38,074 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:38,074 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3932e3d622fc4ee0a4f9d9094244a246, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/f8ce442960d540ff9d9059a4fec64a9c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ed43438af7de4dd393f1224fd9956a84] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=34.4 K 2024-11-18T03:17:38,077 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 423eee99fd4a4c968b66a542a6b2faa6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731899853886 2024-11-18T03:17:38,078 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 3932e3d622fc4ee0a4f9d9094244a246, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731899853101 2024-11-18T03:17:38,081 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de6f911a4964e099584c4c3f18d3eaa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731899856057 2024-11-18T03:17:38,081 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ce442960d540ff9d9059a4fec64a9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731899853886 2024-11-18T03:17:38,085 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ed43438af7de4dd393f1224fd9956a84, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731899856057 2024-11-18T03:17:38,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-18T03:17:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T03:17:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:38,097 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#567 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:38,098 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/05728fe86a754033a99a92b291f61dcd is 50, key is test_row_0/C:col10/1731899856057/Put/seqid=0 2024-11-18T03:17:38,102 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:38,107 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411183756070b1cbf4aee92baef9f0675ff86_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:38,109 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411183756070b1cbf4aee92baef9f0675ff86_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:38,110 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183756070b1cbf4aee92baef9f0675ff86_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:38,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118365823e93913491fbc2408f66769f9e5_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899858088/Put/seqid=0 2024-11-18T03:17:38,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899918119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899918122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899918133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899918138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899918139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742502_1678 (size=4469) 2024-11-18T03:17:38,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742501_1677 (size=13119) 2024-11-18T03:17:38,265 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/05728fe86a754033a99a92b291f61dcd as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/05728fe86a754033a99a92b291f61dcd 2024-11-18T03:17:38,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742503_1679 (size=14994) 2024-11-18T03:17:38,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899918252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899918251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899918264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899918264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899918271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,308 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into 05728fe86a754033a99a92b291f61dcd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:38,308 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:38,308 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=13, startTime=1731899858063; duration=0sec 2024-11-18T03:17:38,308 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:38,309 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:38,309 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:38,324 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:38,324 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:38,324 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:38,325 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5e225a218c1c4d50901f554c3f5f4007, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/313f8cf8a5f04709abc1b76af2b08dec, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/70890188c28a4ba8ae41b4de62160993] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=34.4 K 2024-11-18T03:17:38,328 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e225a218c1c4d50901f554c3f5f4007, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731899853101 2024-11-18T03:17:38,329 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 313f8cf8a5f04709abc1b76af2b08dec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731899853886 2024-11-18T03:17:38,336 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 70890188c28a4ba8ae41b4de62160993, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731899856057 2024-11-18T03:17:38,401 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#570 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:38,401 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/e7b5afa182d84fa68368828d80e3b672 is 50, key is test_row_0/B:col10/1731899856057/Put/seqid=0 2024-11-18T03:17:38,402 INFO [Thread-2580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2580. 2024-11-18T03:17:38,402 DEBUG [Thread-2580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:38,402 DEBUG [Thread-2580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,402 DEBUG [Thread-2580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:38,407 INFO [Thread-2582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2582. 2024-11-18T03:17:38,407 DEBUG [Thread-2582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:38,407 DEBUG [Thread-2582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,408 DEBUG [Thread-2582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,408 INFO [Thread-2584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2584. 2024-11-18T03:17:38,408 INFO [Thread-2588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2588. 2024-11-18T03:17:38,408 DEBUG [Thread-2584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:38,408 DEBUG [Thread-2584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,408 DEBUG [Thread-2584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:38,410 DEBUG [Thread-2588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:38,410 DEBUG [Thread-2588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,410 DEBUG [Thread-2588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:38,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:38,419 INFO [Thread-2586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2586. 2024-11-18T03:17:38,419 DEBUG [Thread-2586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:38,419 DEBUG [Thread-2586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,420 DEBUG [Thread-2586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:38,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:38,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742504_1680 (size=13119) 2024-11-18T03:17:38,439 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/e7b5afa182d84fa68368828d80e3b672 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/e7b5afa182d84fa68368828d80e3b672 2024-11-18T03:17:38,466 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into e7b5afa182d84fa68368828d80e3b672(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:38,466 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:38,466 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=13, startTime=1731899858062; duration=0sec 2024-11-18T03:17:38,466 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:38,466 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:38,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899918481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899918482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899918501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899918501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899918515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,568 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#568 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:38,569 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ae568b0a26274b1faacdf6750a6ef193 is 175, key is test_row_0/A:col10/1731899856057/Put/seqid=0 2024-11-18T03:17:38,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742505_1681 (size=32180) 2024-11-18T03:17:38,597 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/ae568b0a26274b1faacdf6750a6ef193 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae568b0a26274b1faacdf6750a6ef193 2024-11-18T03:17:38,605 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into ae568b0a26274b1faacdf6750a6ef193(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:38,605 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:38,605 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=13, startTime=1731899858062; duration=0sec 2024-11-18T03:17:38,605 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:38,605 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:38,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:38,670 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118365823e93913491fbc2408f66769f9e5_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118365823e93913491fbc2408f66769f9e5_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:38,671 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/39cc99e8dbda43728967932ded750580, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:38,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/39cc99e8dbda43728967932ded750580 is 175, key is test_row_0/A:col10/1731899858088/Put/seqid=0 2024-11-18T03:17:38,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742506_1682 (size=39949) 2024-11-18T03:17:38,682 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/39cc99e8dbda43728967932ded750580 2024-11-18T03:17:38,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/ab745f6b9c2a4b299924dfe1076011d3 is 50, key is test_row_0/B:col10/1731899858088/Put/seqid=0 2024-11-18T03:17:38,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742507_1683 (size=12301) 2024-11-18T03:17:38,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/ab745f6b9c2a4b299924dfe1076011d3 2024-11-18T03:17:38,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/dca7a0f946e542a88981f63abcb8d336 is 50, key is test_row_0/C:col10/1731899858088/Put/seqid=0 2024-11-18T03:17:38,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39116 deadline: 1731899918795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39170 deadline: 1731899918794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742508_1684 (size=12301) 2024-11-18T03:17:38,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39182 deadline: 1731899918821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39108 deadline: 1731899918828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T03:17:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39122 deadline: 1731899918831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 , the old value is region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T03:17:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., hostname=fd862c3a04f9,38549,1731899684479, seqNum=5 because the exception is null or not the one we care about 2024-11-18T03:17:39,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/dca7a0f946e542a88981f63abcb8d336 2024-11-18T03:17:39,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/39cc99e8dbda43728967932ded750580 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/39cc99e8dbda43728967932ded750580 2024-11-18T03:17:39,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/39cc99e8dbda43728967932ded750580, entries=200, sequenceid=431, filesize=39.0 K 2024-11-18T03:17:39,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/ab745f6b9c2a4b299924dfe1076011d3 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ab745f6b9c2a4b299924dfe1076011d3 2024-11-18T03:17:39,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ab745f6b9c2a4b299924dfe1076011d3, entries=150, sequenceid=431, filesize=12.0 K 2024-11-18T03:17:39,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/dca7a0f946e542a88981f63abcb8d336 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/dca7a0f946e542a88981f63abcb8d336 2024-11-18T03:17:39,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/dca7a0f946e542a88981f63abcb8d336, entries=150, sequenceid=431, filesize=12.0 K 2024-11-18T03:17:39,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=60.38 KB/61830 for 66ab176d63aa2fde549e454fab5778b4 in 1131ms, sequenceid=431, compaction requested=false 2024-11-18T03:17:39,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38549 {}] regionserver.HRegion(8855): Flush requested on 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:39,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T03:17:39,303 INFO [Thread-2569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2569. 2024-11-18T03:17:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:39,303 DEBUG [Thread-2569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:39,303 DEBUG [Thread-2569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:39,303 DEBUG [Thread-2569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:39,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:39,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bba5c8ac4b194d4ea7f15ad069d7d19f_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899858134/Put/seqid=0 2024-11-18T03:17:39,316 INFO [Thread-2573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2573. 2024-11-18T03:17:39,316 DEBUG [Thread-2573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:39,316 DEBUG [Thread-2573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,316 DEBUG [Thread-2573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:39,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742509_1685 (size=12454) 2024-11-18T03:17:39,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:39,333 INFO [Thread-2571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2571. 2024-11-18T03:17:39,333 DEBUG [Thread-2571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:39,333 DEBUG [Thread-2571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,333 DEBUG [Thread-2571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:39,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bba5c8ac4b194d4ea7f15ad069d7d19f_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bba5c8ac4b194d4ea7f15ad069d7d19f_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:39,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/19324ddfb96f4a13802f1b009e4ee956, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:39,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/19324ddfb96f4a13802f1b009e4ee956 is 175, key is test_row_0/A:col10/1731899858134/Put/seqid=0 2024-11-18T03:17:39,343 INFO [Thread-2575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2575. 2024-11-18T03:17:39,343 DEBUG [Thread-2575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:39,343 DEBUG [Thread-2575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,343 DEBUG [Thread-2575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:39,343 INFO [Thread-2577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2577. 2024-11-18T03:17:39,343 DEBUG [Thread-2577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T03:17:39,343 DEBUG [Thread-2577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,343 DEBUG [Thread-2577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:39,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:39,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742510_1686 (size=31255) 2024-11-18T03:17:39,749 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/19324ddfb96f4a13802f1b009e4ee956 2024-11-18T03:17:39,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/37c7fb8c7c3d4977bd8c1227cb417e58 is 50, key is test_row_0/B:col10/1731899858134/Put/seqid=0 2024-11-18T03:17:39,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742511_1687 (size=12301) 2024-11-18T03:17:39,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/37c7fb8c7c3d4977bd8c1227cb417e58 2024-11-18T03:17:39,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/97944a1c33304353b6c73c78c177b932 is 50, key is test_row_0/C:col10/1731899858134/Put/seqid=0 2024-11-18T03:17:39,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742512_1688 (size=12301) 2024-11-18T03:17:39,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/97944a1c33304353b6c73c78c177b932 2024-11-18T03:17:39,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/19324ddfb96f4a13802f1b009e4ee956 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/19324ddfb96f4a13802f1b009e4ee956 2024-11-18T03:17:39,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/19324ddfb96f4a13802f1b009e4ee956, entries=150, sequenceid=447, filesize=30.5 K 2024-11-18T03:17:39,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/37c7fb8c7c3d4977bd8c1227cb417e58 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/37c7fb8c7c3d4977bd8c1227cb417e58 2024-11-18T03:17:39,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/37c7fb8c7c3d4977bd8c1227cb417e58, entries=150, sequenceid=447, filesize=12.0 K 2024-11-18T03:17:39,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/97944a1c33304353b6c73c78c177b932 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/97944a1c33304353b6c73c78c177b932 2024-11-18T03:17:39,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/97944a1c33304353b6c73c78c177b932, entries=150, sequenceid=447, filesize=12.0 K 2024-11-18T03:17:39,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 66ab176d63aa2fde549e454fab5778b4 in 616ms, sequenceid=447, compaction requested=true 2024-11-18T03:17:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T03:17:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:39,919 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T03:17:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:39,919 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66ab176d63aa2fde549e454fab5778b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T03:17:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:39,921 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103384 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:39,921 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/A is initiating minor compaction (all files) 2024-11-18T03:17:39,921 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/A in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:39,921 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae568b0a26274b1faacdf6750a6ef193, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/39cc99e8dbda43728967932ded750580, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/19324ddfb96f4a13802f1b009e4ee956] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=101.0 K 2024-11-18T03:17:39,921 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:39,921 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. files: [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae568b0a26274b1faacdf6750a6ef193, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/39cc99e8dbda43728967932ded750580, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/19324ddfb96f4a13802f1b009e4ee956] 2024-11-18T03:17:39,922 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting ae568b0a26274b1faacdf6750a6ef193, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731899853890 2024-11-18T03:17:39,923 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 39cc99e8dbda43728967932ded750580, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731899857423 2024-11-18T03:17:39,923 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:39,923 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] compactions.Compactor(225): Compacting 19324ddfb96f4a13802f1b009e4ee956, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731899858117 2024-11-18T03:17:39,923 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/B is initiating minor compaction (all files) 2024-11-18T03:17:39,923 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/B in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:39,924 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/e7b5afa182d84fa68368828d80e3b672, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ab745f6b9c2a4b299924dfe1076011d3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/37c7fb8c7c3d4977bd8c1227cb417e58] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=36.8 K 2024-11-18T03:17:39,925 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b5afa182d84fa68368828d80e3b672, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731899853890 2024-11-18T03:17:39,926 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab745f6b9c2a4b299924dfe1076011d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731899857434 2024-11-18T03:17:39,926 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37c7fb8c7c3d4977bd8c1227cb417e58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731899858117 2024-11-18T03:17:39,961 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:39,963 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#B#compaction#576 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:39,963 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/a9bda7705e8448f09256e4dad84c19c6 is 50, key is test_row_0/B:col10/1731899858134/Put/seqid=0 2024-11-18T03:17:39,975 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411188e6253269fcf4783bcb63c5e0661c610_66ab176d63aa2fde549e454fab5778b4 store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:39,978 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411188e6253269fcf4783bcb63c5e0661c610_66ab176d63aa2fde549e454fab5778b4, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:39,978 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188e6253269fcf4783bcb63c5e0661c610_66ab176d63aa2fde549e454fab5778b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:39,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742513_1689 (size=13221) 2024-11-18T03:17:39,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742514_1690 (size=4469) 2024-11-18T03:17:40,390 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/a9bda7705e8448f09256e4dad84c19c6 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a9bda7705e8448f09256e4dad84c19c6 2024-11-18T03:17:40,400 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#A#compaction#577 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:40,401 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/c30839f36bcd49079cee45891353a1aa is 175, key is test_row_0/A:col10/1731899858134/Put/seqid=0 2024-11-18T03:17:40,407 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/B of 66ab176d63aa2fde549e454fab5778b4 into a9bda7705e8448f09256e4dad84c19c6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:40,407 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:40,407 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/B, priority=13, startTime=1731899859919; duration=0sec 2024-11-18T03:17:40,407 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T03:17:40,407 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:B 2024-11-18T03:17:40,407 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T03:17:40,408 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T03:17:40,408 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1541): 66ab176d63aa2fde549e454fab5778b4/C is initiating minor compaction (all files) 2024-11-18T03:17:40,408 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66ab176d63aa2fde549e454fab5778b4/C in TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:40,409 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/05728fe86a754033a99a92b291f61dcd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/dca7a0f946e542a88981f63abcb8d336, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/97944a1c33304353b6c73c78c177b932] into tmpdir=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp, totalSize=36.8 K 2024-11-18T03:17:40,412 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05728fe86a754033a99a92b291f61dcd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731899853890 2024-11-18T03:17:40,413 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca7a0f946e542a88981f63abcb8d336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731899857434 2024-11-18T03:17:40,413 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97944a1c33304353b6c73c78c177b932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731899858117 2024-11-18T03:17:40,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742515_1691 (size=32175) 2024-11-18T03:17:40,424 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66ab176d63aa2fde549e454fab5778b4#C#compaction#578 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T03:17:40,425 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/23d95b951e054ccab9fbdb20208d3339 is 50, key is test_row_0/C:col10/1731899858134/Put/seqid=0 2024-11-18T03:17:40,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742516_1692 (size=13221) 2024-11-18T03:17:40,818 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/c30839f36bcd49079cee45891353a1aa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c30839f36bcd49079cee45891353a1aa 2024-11-18T03:17:40,822 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/A of 66ab176d63aa2fde549e454fab5778b4 into c30839f36bcd49079cee45891353a1aa(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:40,822 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:40,822 INFO [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/A, priority=13, startTime=1731899859919; duration=0sec 2024-11-18T03:17:40,822 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:40,822 DEBUG [RS:0;fd862c3a04f9:38549-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:A 2024-11-18T03:17:40,845 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/23d95b951e054ccab9fbdb20208d3339 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/23d95b951e054ccab9fbdb20208d3339 2024-11-18T03:17:40,849 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66ab176d63aa2fde549e454fab5778b4/C of 66ab176d63aa2fde549e454fab5778b4 into 23d95b951e054ccab9fbdb20208d3339(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T03:17:40,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66ab176d63aa2fde549e454fab5778b4: 2024-11-18T03:17:40,849 INFO [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., storeName=66ab176d63aa2fde549e454fab5778b4/C, priority=13, startTime=1731899859919; duration=0sec 2024-11-18T03:17:40,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T03:17:40,849 DEBUG [RS:0;fd862c3a04f9:38549-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66ab176d63aa2fde549e454fab5778b4:C 2024-11-18T03:17:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T03:17:41,042 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1929 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1734 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1828 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1936 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1729 2024-11-18T03:17:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T03:17:41,042 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:17:41,042 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:17:41,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:41,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:41,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:41,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-18T03:17:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T03:17:41,054 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899861054"}]},"ts":"1731899861054"} 2024-11-18T03:17:41,056 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T03:17:41,056 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T03:17:41,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T03:17:41,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, UNASSIGN}] 2024-11-18T03:17:41,060 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, UNASSIGN 2024-11-18T03:17:41,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=CLOSING, regionLocation=fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:41,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, UNASSIGN because future has completed 2024-11-18T03:17:41,063 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T03:17:41,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479}] 2024-11-18T03:17:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T03:17:41,216 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 66ab176d63aa2fde549e454fab5778b4, disabling compactions & flushes 2024-11-18T03:17:41,216 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. after waiting 0 ms 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:41,216 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 66ab176d63aa2fde549e454fab5778b4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=A 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:41,216 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=B 2024-11-18T03:17:41,217 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:41,217 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66ab176d63aa2fde549e454fab5778b4, store=C 2024-11-18T03:17:41,217 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T03:17:41,258 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118981ce366435243a0afe6987a12efb8a9_66ab176d63aa2fde549e454fab5778b4 is 50, key is test_row_0/A:col10/1731899859315/Put/seqid=0 2024-11-18T03:17:41,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742517_1693 (size=12454) 2024-11-18T03:17:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T03:17:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T03:17:41,708 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:17:41,735 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118981ce366435243a0afe6987a12efb8a9_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118981ce366435243a0afe6987a12efb8a9_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:41,736 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f47670bdb84545e0865bc7fc6d49c462, store: [table=TestAcidGuarantees family=A region=66ab176d63aa2fde549e454fab5778b4] 2024-11-18T03:17:41,737 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f47670bdb84545e0865bc7fc6d49c462 is 175, key is test_row_0/A:col10/1731899859315/Put/seqid=0 2024-11-18T03:17:41,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742518_1694 (size=31255) 2024-11-18T03:17:41,745 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f47670bdb84545e0865bc7fc6d49c462 2024-11-18T03:17:41,751 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/d591971fa48449d5bee3088262b69344 is 50, key is test_row_0/B:col10/1731899859315/Put/seqid=0 2024-11-18T03:17:41,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742519_1695 (size=12301) 2024-11-18T03:17:41,754 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/d591971fa48449d5bee3088262b69344 2024-11-18T03:17:41,759 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/1d81015777294a48803e74de71d635fa is 50, key is test_row_0/C:col10/1731899859315/Put/seqid=0 2024-11-18T03:17:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742520_1696 (size=12301) 2024-11-18T03:17:42,179 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/1d81015777294a48803e74de71d635fa 2024-11-18T03:17:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T03:17:42,208 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/A/f47670bdb84545e0865bc7fc6d49c462 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f47670bdb84545e0865bc7fc6d49c462 2024-11-18T03:17:42,215 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f47670bdb84545e0865bc7fc6d49c462, entries=150, sequenceid=457, filesize=30.5 K 2024-11-18T03:17:42,217 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T03:17:42,220 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/B/d591971fa48449d5bee3088262b69344 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d591971fa48449d5bee3088262b69344 2024-11-18T03:17:42,227 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d591971fa48449d5bee3088262b69344, entries=150, sequenceid=457, filesize=12.0 K 2024-11-18T03:17:42,228 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/.tmp/C/1d81015777294a48803e74de71d635fa as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/1d81015777294a48803e74de71d635fa 2024-11-18T03:17:42,237 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/1d81015777294a48803e74de71d635fa, entries=150, sequenceid=457, filesize=12.0 K 2024-11-18T03:17:42,244 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 66ab176d63aa2fde549e454fab5778b4 in 1028ms, sequenceid=457, compaction requested=false 2024-11-18T03:17:42,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/153e61cbd475463181aa0cf21c4d2225, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f76375bb6ce043c29db4dae60dd05087, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/de05e1c9f65d4a5599e3d8f9467960f7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd6a50bc83324f87b515eade84de7af8, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d26a2c74203440369a7ca6845c7c9aba, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/73745087302d4d26bd20a95098c9317e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/acb7ee0c4bc8452fbb32ed482ba10b36, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ce133a22603c44828cf8e4a241594962, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/4538b9e12047475f83597c3cfe7452f1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/7cc8973734e5432c87902eabdd131aa1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/b26ed93145d94fcba7fd0176ba853523, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae0f0790f22f4485b037930d6e7a4867, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/fdbb6069e2d44408b1ca6ccd04d2b7d5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/e65842850c054cad9985faa7acd6fe6b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd8f891955164b788166f80b43072bb1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d41b1682de7f456abfa486639262c2c4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/0fe671c0dad44e7eaecb1ec905b14993, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/bc16e46b165c4b229a7c7b8d1b7dda43, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d1bd596e028f481b8fc4b3950a7f6e43, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2fab54b920ec4c29a7e58c377526a6f0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c33dd5215f02448c8ac1e96b36bf730e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/a4bb6ad141644be1855f5949877f37d0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/423eee99fd4a4c968b66a542a6b2faa6, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae568b0a26274b1faacdf6750a6ef193, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2de6f911a4964e099584c4c3f18d3eaa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/39cc99e8dbda43728967932ded750580, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/19324ddfb96f4a13802f1b009e4ee956] to archive 2024-11-18T03:17:42,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:17:42,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/153e61cbd475463181aa0cf21c4d2225 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/153e61cbd475463181aa0cf21c4d2225 2024-11-18T03:17:42,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f76375bb6ce043c29db4dae60dd05087 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f76375bb6ce043c29db4dae60dd05087 2024-11-18T03:17:42,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/de05e1c9f65d4a5599e3d8f9467960f7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/de05e1c9f65d4a5599e3d8f9467960f7 2024-11-18T03:17:42,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd6a50bc83324f87b515eade84de7af8 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd6a50bc83324f87b515eade84de7af8 2024-11-18T03:17:42,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d26a2c74203440369a7ca6845c7c9aba to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d26a2c74203440369a7ca6845c7c9aba 2024-11-18T03:17:42,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/73745087302d4d26bd20a95098c9317e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/73745087302d4d26bd20a95098c9317e 2024-11-18T03:17:42,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/acb7ee0c4bc8452fbb32ed482ba10b36 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/acb7ee0c4bc8452fbb32ed482ba10b36 2024-11-18T03:17:42,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ce133a22603c44828cf8e4a241594962 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ce133a22603c44828cf8e4a241594962 2024-11-18T03:17:42,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/4538b9e12047475f83597c3cfe7452f1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/4538b9e12047475f83597c3cfe7452f1 2024-11-18T03:17:42,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/7cc8973734e5432c87902eabdd131aa1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/7cc8973734e5432c87902eabdd131aa1 2024-11-18T03:17:42,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/b26ed93145d94fcba7fd0176ba853523 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/b26ed93145d94fcba7fd0176ba853523 2024-11-18T03:17:42,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae0f0790f22f4485b037930d6e7a4867 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae0f0790f22f4485b037930d6e7a4867 2024-11-18T03:17:42,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/fdbb6069e2d44408b1ca6ccd04d2b7d5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/fdbb6069e2d44408b1ca6ccd04d2b7d5 2024-11-18T03:17:42,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/e65842850c054cad9985faa7acd6fe6b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/e65842850c054cad9985faa7acd6fe6b 2024-11-18T03:17:42,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd8f891955164b788166f80b43072bb1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/dd8f891955164b788166f80b43072bb1 2024-11-18T03:17:42,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d41b1682de7f456abfa486639262c2c4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d41b1682de7f456abfa486639262c2c4 2024-11-18T03:17:42,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/0fe671c0dad44e7eaecb1ec905b14993 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/0fe671c0dad44e7eaecb1ec905b14993 2024-11-18T03:17:42,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/bc16e46b165c4b229a7c7b8d1b7dda43 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/bc16e46b165c4b229a7c7b8d1b7dda43 2024-11-18T03:17:42,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d1bd596e028f481b8fc4b3950a7f6e43 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/d1bd596e028f481b8fc4b3950a7f6e43 2024-11-18T03:17:42,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2fab54b920ec4c29a7e58c377526a6f0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2fab54b920ec4c29a7e58c377526a6f0 2024-11-18T03:17:42,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c33dd5215f02448c8ac1e96b36bf730e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c33dd5215f02448c8ac1e96b36bf730e 2024-11-18T03:17:42,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/a4bb6ad141644be1855f5949877f37d0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/a4bb6ad141644be1855f5949877f37d0 2024-11-18T03:17:42,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/423eee99fd4a4c968b66a542a6b2faa6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/423eee99fd4a4c968b66a542a6b2faa6 2024-11-18T03:17:42,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae568b0a26274b1faacdf6750a6ef193 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/ae568b0a26274b1faacdf6750a6ef193 2024-11-18T03:17:42,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2de6f911a4964e099584c4c3f18d3eaa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/2de6f911a4964e099584c4c3f18d3eaa 2024-11-18T03:17:42,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/39cc99e8dbda43728967932ded750580 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/39cc99e8dbda43728967932ded750580 2024-11-18T03:17:42,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/19324ddfb96f4a13802f1b009e4ee956 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/19324ddfb96f4a13802f1b009e4ee956 2024-11-18T03:17:42,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ed53ab5c9cfd47558cc77acc1007280a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5d1914bbfa7643eb802a90a7dbd5597b, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f3fe63f5ba7d4592ae64d9c73c9459f2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6392bbf3c6c04223b5bf931af125e3c1, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f7ab8971db3b4c93bf7f393e46072a95, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a099cb973ed84b069490e96741f13784, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/92a853056d1742bf8052c8140c68a309, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/67b5613f193e471b8ea38b39b6ba4c3a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/82359a3cc5784cf7a2e09e80240a99d9, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d46dae9f54ea4dc191f6cb189678c4a7, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6bf2bbfba66e479bb0a3321397facdce, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/fe2da8f71c844bebabdd9a0eec80d13c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/3b5d834638804bfab7dfaaeeb155ff8e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/8a713ceec9e840468fe3427d192d8bcd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c791105f61a24bb88e6aff13348d5b0e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/567e71bd61814c11998188fe53b68a27, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f1135e05982748d68ed8f936755e116d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/331a6f31afb34dbc872edea1cc1fbddd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/108219d8e3254c38af38df6dd58bffac, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/30a0bcca26374fa58d88ee358309be10, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5e225a218c1c4d50901f554c3f5f4007, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c3d70a3ed015456582b57c1048abc1e2, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/313f8cf8a5f04709abc1b76af2b08dec, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/e7b5afa182d84fa68368828d80e3b672, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/70890188c28a4ba8ae41b4de62160993, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ab745f6b9c2a4b299924dfe1076011d3, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/37c7fb8c7c3d4977bd8c1227cb417e58] to archive 2024-11-18T03:17:42,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:17:42,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ed53ab5c9cfd47558cc77acc1007280a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ed53ab5c9cfd47558cc77acc1007280a 2024-11-18T03:17:42,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5d1914bbfa7643eb802a90a7dbd5597b to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5d1914bbfa7643eb802a90a7dbd5597b 2024-11-18T03:17:42,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f3fe63f5ba7d4592ae64d9c73c9459f2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f3fe63f5ba7d4592ae64d9c73c9459f2 2024-11-18T03:17:42,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6392bbf3c6c04223b5bf931af125e3c1 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6392bbf3c6c04223b5bf931af125e3c1 2024-11-18T03:17:42,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f7ab8971db3b4c93bf7f393e46072a95 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f7ab8971db3b4c93bf7f393e46072a95 2024-11-18T03:17:42,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a099cb973ed84b069490e96741f13784 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a099cb973ed84b069490e96741f13784 2024-11-18T03:17:42,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/92a853056d1742bf8052c8140c68a309 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/92a853056d1742bf8052c8140c68a309 2024-11-18T03:17:42,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/67b5613f193e471b8ea38b39b6ba4c3a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/67b5613f193e471b8ea38b39b6ba4c3a 2024-11-18T03:17:42,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/82359a3cc5784cf7a2e09e80240a99d9 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/82359a3cc5784cf7a2e09e80240a99d9 2024-11-18T03:17:42,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d46dae9f54ea4dc191f6cb189678c4a7 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d46dae9f54ea4dc191f6cb189678c4a7 2024-11-18T03:17:42,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6bf2bbfba66e479bb0a3321397facdce to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/6bf2bbfba66e479bb0a3321397facdce 2024-11-18T03:17:42,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/fe2da8f71c844bebabdd9a0eec80d13c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/fe2da8f71c844bebabdd9a0eec80d13c 2024-11-18T03:17:42,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/3b5d834638804bfab7dfaaeeb155ff8e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/3b5d834638804bfab7dfaaeeb155ff8e 2024-11-18T03:17:42,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/8a713ceec9e840468fe3427d192d8bcd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/8a713ceec9e840468fe3427d192d8bcd 2024-11-18T03:17:42,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c791105f61a24bb88e6aff13348d5b0e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c791105f61a24bb88e6aff13348d5b0e 2024-11-18T03:17:42,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/567e71bd61814c11998188fe53b68a27 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/567e71bd61814c11998188fe53b68a27 2024-11-18T03:17:42,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f1135e05982748d68ed8f936755e116d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/f1135e05982748d68ed8f936755e116d 2024-11-18T03:17:42,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/331a6f31afb34dbc872edea1cc1fbddd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/331a6f31afb34dbc872edea1cc1fbddd 2024-11-18T03:17:42,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/108219d8e3254c38af38df6dd58bffac to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/108219d8e3254c38af38df6dd58bffac 2024-11-18T03:17:42,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/30a0bcca26374fa58d88ee358309be10 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/30a0bcca26374fa58d88ee358309be10 2024-11-18T03:17:42,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5e225a218c1c4d50901f554c3f5f4007 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/5e225a218c1c4d50901f554c3f5f4007 2024-11-18T03:17:42,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c3d70a3ed015456582b57c1048abc1e2 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/c3d70a3ed015456582b57c1048abc1e2 2024-11-18T03:17:42,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/313f8cf8a5f04709abc1b76af2b08dec to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/313f8cf8a5f04709abc1b76af2b08dec 2024-11-18T03:17:42,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/e7b5afa182d84fa68368828d80e3b672 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/e7b5afa182d84fa68368828d80e3b672 2024-11-18T03:17:42,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/70890188c28a4ba8ae41b4de62160993 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/70890188c28a4ba8ae41b4de62160993 2024-11-18T03:17:42,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ab745f6b9c2a4b299924dfe1076011d3 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/ab745f6b9c2a4b299924dfe1076011d3 2024-11-18T03:17:42,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/37c7fb8c7c3d4977bd8c1227cb417e58 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/37c7fb8c7c3d4977bd8c1227cb417e58 2024-11-18T03:17:42,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6a468398b9d445a0ab0bea3928f4087c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/94e4807c977d47e5a8920fe3a89d31f0, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e7f895c91ea7447faddecdd7783f8642, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/5f9b6e42a25b4617b3c306cd4ccee75f, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e41de585d5dd484b9e5cfa4111ffaefe, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/d0bdbe4e938749668e236c492b14fd90, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/459f13da0b074c708132cd82595dd5a4, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/305c4eedd62e41cea6fd93c964939dfa, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ef9067637c7b41388437bf91685b9f7d, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6914334ebe014c139b6edb17f60060b5, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e9e4195e5a304f37b5c85e6c2bf63e3a, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/944615574edb40b6a8e4209a53d8a836, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/553fa051979f4221aa0e8acde6c57418, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/a649045ecb4f4f4dacee118f84716053, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/910df35cefb647eb83b38f5ae742b529, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e59bb64b653f4415b89d9d3c84767298, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/38c66ec6e0e64530b12af5d34203a026, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/41cbcfd4c2524f17b0382018cb55bcdc, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3d8c40ca689444cea1dd057577c9f785, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/0dccef52e85943caa06a1e53c774ae15, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3932e3d622fc4ee0a4f9d9094244a246, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/c1a20e3d48bc41da9012983eae38b38e, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/f8ce442960d540ff9d9059a4fec64a9c, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/05728fe86a754033a99a92b291f61dcd, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ed43438af7de4dd393f1224fd9956a84, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/dca7a0f946e542a88981f63abcb8d336, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/97944a1c33304353b6c73c78c177b932] to archive 2024-11-18T03:17:42,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T03:17:42,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6a468398b9d445a0ab0bea3928f4087c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6a468398b9d445a0ab0bea3928f4087c 2024-11-18T03:17:42,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/94e4807c977d47e5a8920fe3a89d31f0 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/94e4807c977d47e5a8920fe3a89d31f0 2024-11-18T03:17:42,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e7f895c91ea7447faddecdd7783f8642 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e7f895c91ea7447faddecdd7783f8642 2024-11-18T03:17:42,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/5f9b6e42a25b4617b3c306cd4ccee75f to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/5f9b6e42a25b4617b3c306cd4ccee75f 2024-11-18T03:17:42,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e41de585d5dd484b9e5cfa4111ffaefe to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e41de585d5dd484b9e5cfa4111ffaefe 2024-11-18T03:17:42,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/d0bdbe4e938749668e236c492b14fd90 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/d0bdbe4e938749668e236c492b14fd90 2024-11-18T03:17:42,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/459f13da0b074c708132cd82595dd5a4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/459f13da0b074c708132cd82595dd5a4 2024-11-18T03:17:42,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/305c4eedd62e41cea6fd93c964939dfa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/305c4eedd62e41cea6fd93c964939dfa 2024-11-18T03:17:42,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ef9067637c7b41388437bf91685b9f7d to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ef9067637c7b41388437bf91685b9f7d 2024-11-18T03:17:42,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6914334ebe014c139b6edb17f60060b5 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/6914334ebe014c139b6edb17f60060b5 2024-11-18T03:17:42,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e9e4195e5a304f37b5c85e6c2bf63e3a to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e9e4195e5a304f37b5c85e6c2bf63e3a 2024-11-18T03:17:42,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/944615574edb40b6a8e4209a53d8a836 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/944615574edb40b6a8e4209a53d8a836 2024-11-18T03:17:42,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/553fa051979f4221aa0e8acde6c57418 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/553fa051979f4221aa0e8acde6c57418 2024-11-18T03:17:42,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/a649045ecb4f4f4dacee118f84716053 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/a649045ecb4f4f4dacee118f84716053 2024-11-18T03:17:42,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/910df35cefb647eb83b38f5ae742b529 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/910df35cefb647eb83b38f5ae742b529 2024-11-18T03:17:42,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e59bb64b653f4415b89d9d3c84767298 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/e59bb64b653f4415b89d9d3c84767298 2024-11-18T03:17:42,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/38c66ec6e0e64530b12af5d34203a026 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/38c66ec6e0e64530b12af5d34203a026 2024-11-18T03:17:42,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/41cbcfd4c2524f17b0382018cb55bcdc to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/41cbcfd4c2524f17b0382018cb55bcdc 2024-11-18T03:17:42,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3d8c40ca689444cea1dd057577c9f785 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3d8c40ca689444cea1dd057577c9f785 2024-11-18T03:17:42,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/0dccef52e85943caa06a1e53c774ae15 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/0dccef52e85943caa06a1e53c774ae15 2024-11-18T03:17:42,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3932e3d622fc4ee0a4f9d9094244a246 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/3932e3d622fc4ee0a4f9d9094244a246 2024-11-18T03:17:42,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/c1a20e3d48bc41da9012983eae38b38e to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/c1a20e3d48bc41da9012983eae38b38e 2024-11-18T03:17:42,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/f8ce442960d540ff9d9059a4fec64a9c to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/f8ce442960d540ff9d9059a4fec64a9c 2024-11-18T03:17:42,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/05728fe86a754033a99a92b291f61dcd to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/05728fe86a754033a99a92b291f61dcd 2024-11-18T03:17:42,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ed43438af7de4dd393f1224fd9956a84 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/ed43438af7de4dd393f1224fd9956a84 2024-11-18T03:17:42,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/dca7a0f946e542a88981f63abcb8d336 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/dca7a0f946e542a88981f63abcb8d336 2024-11-18T03:17:42,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/97944a1c33304353b6c73c78c177b932 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/97944a1c33304353b6c73c78c177b932 2024-11-18T03:17:42,453 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/recovered.edits/460.seqid, newMaxSeqId=460, maxSeqId=4 2024-11-18T03:17:42,453 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. 2024-11-18T03:17:42,453 DEBUG [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 66ab176d63aa2fde549e454fab5778b4: Waiting for close lock at 1731899861216Running coprocessor pre-close hooks at 1731899861216Disabling compacts and flushes for region at 1731899861216Disabling writes for close at 1731899861216Obtaining lock to block concurrent updates at 1731899861216Preparing flush snapshotting stores in 66ab176d63aa2fde549e454fab5778b4 at 1731899861216Finished memstore snapshotting TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731899861217 (+1 ms)Flushing stores of TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4. at 1731899861218 (+1 ms)Flushing 66ab176d63aa2fde549e454fab5778b4/A: creating writer at 1731899861218Flushing mob file 66ab176d63aa2fde549e454fab5778b4/A: appending metadata at 1731899861258 (+40 ms)Flushing mob file 66ab176d63aa2fde549e454fab5778b4/A: closing flushed file at 1731899861258Flushing 66ab176d63aa2fde549e454fab5778b4/A: appending metadata at 1731899861736 (+478 ms)Flushing 66ab176d63aa2fde549e454fab5778b4/A: closing flushed file at 1731899861736Flushing 66ab176d63aa2fde549e454fab5778b4/B: creating writer at 1731899861748 (+12 ms)Flushing 66ab176d63aa2fde549e454fab5778b4/B: appending metadata at 1731899861750 (+2 ms)Flushing 66ab176d63aa2fde549e454fab5778b4/B: closing flushed file at 1731899861750Flushing 66ab176d63aa2fde549e454fab5778b4/C: creating writer at 1731899861756 (+6 ms)Flushing 66ab176d63aa2fde549e454fab5778b4/C: appending metadata at 1731899861758 (+2 ms)Flushing 66ab176d63aa2fde549e454fab5778b4/C: closing flushed file at 1731899861758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54423eef: reopening flushed file at 1731899862207 (+449 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@642b130e: reopening flushed file at 1731899862215 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@324ee8e3: reopening flushed file at 1731899862227 (+12 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 66ab176d63aa2fde549e454fab5778b4 in 1028ms, sequenceid=457, compaction requested=false at 1731899862244 (+17 ms)Writing region close event to WAL at 1731899862449 (+205 ms)Running coprocessor post-close hooks at 1731899862453 (+4 ms)Closed at 1731899862453 2024-11-18T03:17:42,455 INFO [RS_CLOSE_REGION-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:42,455 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=66ab176d63aa2fde549e454fab5778b4, regionState=CLOSED 2024-11-18T03:17:42,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 because future has completed 2024-11-18T03:17:42,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-18T03:17:42,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 66ab176d63aa2fde549e454fab5778b4, server=fd862c3a04f9,38549,1731899684479 in 1.3940 sec 2024-11-18T03:17:42,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-18T03:17:42,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66ab176d63aa2fde549e454fab5778b4, UNASSIGN in 1.4000 sec 2024-11-18T03:17:42,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-18T03:17:42,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4030 sec 2024-11-18T03:17:42,463 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731899862463"}]},"ts":"1731899862463"} 2024-11-18T03:17:42,464 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T03:17:42,464 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T03:17:42,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4210 sec 2024-11-18T03:17:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T03:17:43,201 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:43,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-18T03:17:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:43,202 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-18T03:17:43,203 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:43,205 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,206 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C, FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/recovered.edits] 2024-11-18T03:17:43,208 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c30839f36bcd49079cee45891353a1aa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/c30839f36bcd49079cee45891353a1aa 2024-11-18T03:17:43,209 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f47670bdb84545e0865bc7fc6d49c462 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/A/f47670bdb84545e0865bc7fc6d49c462 2024-11-18T03:17:43,211 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a9bda7705e8448f09256e4dad84c19c6 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/a9bda7705e8448f09256e4dad84c19c6 2024-11-18T03:17:43,212 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d591971fa48449d5bee3088262b69344 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/B/d591971fa48449d5bee3088262b69344 2024-11-18T03:17:43,214 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/1d81015777294a48803e74de71d635fa to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/1d81015777294a48803e74de71d635fa 2024-11-18T03:17:43,214 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/23d95b951e054ccab9fbdb20208d3339 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/C/23d95b951e054ccab9fbdb20208d3339 2024-11-18T03:17:43,216 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/recovered.edits/460.seqid to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4/recovered.edits/460.seqid 2024-11-18T03:17:43,216 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/default/TestAcidGuarantees/66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,216 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T03:17:43,217 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T03:17:43,217 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T03:17:43,220 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180c50dfa38fbf4ef1a01ae3cf6650aefb_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180c50dfa38fbf4ef1a01ae3cf6650aefb_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,221 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118179ebff8890f463f8d7de19299075ed0_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118179ebff8890f463f8d7de19299075ed0_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,221 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182067010a7eac4fa99ae86d245585e078_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182067010a7eac4fa99ae86d245585e078_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,222 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828dd8a373f924432852838f0f6d66183_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828dd8a373f924432852838f0f6d66183_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,223 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118291be27521574ff79ef7e3d805b52e22_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118291be27521574ff79ef7e3d805b52e22_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111829f0c7d0b0fe4d628deffbb2bb8d2bd5_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111829f0c7d0b0fe4d628deffbb2bb8d2bd5_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182c8b4a4294c54f9e9570d5f550798065_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182c8b4a4294c54f9e9570d5f550798065_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,225 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118365823e93913491fbc2408f66769f9e5_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118365823e93913491fbc2408f66769f9e5_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,226 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118431464dbfc884a2cb01dbaab0126d710_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118431464dbfc884a2cb01dbaab0126d710_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,227 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111859d1fee34d9744f99f97b42b6059c12a_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111859d1fee34d9744f99f97b42b6059c12a_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,228 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118605a4ef594d1470887a6a1fd016b3f44_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118605a4ef594d1470887a6a1fd016b3f44_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,229 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118638399e09548474d8443d50f530fe43b_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118638399e09548474d8443d50f530fe43b_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,229 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118783c55f60a2d484abdd2a19db5bd63b8_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118783c55f60a2d484abdd2a19db5bd63b8_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,230 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187a63b79024814eb3801810d992bc7f49_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187a63b79024814eb3801810d992bc7f49_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,231 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189324b376d6754947903910b049a6b834_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189324b376d6754947903910b049a6b834_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,232 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118981ce366435243a0afe6987a12efb8a9_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118981ce366435243a0afe6987a12efb8a9_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,233 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189834cc51ee1d4dbea7bc8ba8788613e7_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189834cc51ee1d4dbea7bc8ba8788613e7_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,233 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bba5c8ac4b194d4ea7f15ad069d7d19f_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bba5c8ac4b194d4ea7f15ad069d7d19f_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,234 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c2f9146f0bdf421bbd1ec265120088e8_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c2f9146f0bdf421bbd1ec265120088e8_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,235 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e077f81fe1c74b5a823cd24c0010ea3e_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e077f81fe1c74b5a823cd24c0010ea3e_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,235 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fc8353c0b91b4ecb9515a760af93f95f_66ab176d63aa2fde549e454fab5778b4 to hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fc8353c0b91b4ecb9515a760af93f95f_66ab176d63aa2fde549e454fab5778b4 2024-11-18T03:17:43,236 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T03:17:43,238 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:43,240 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T03:17:43,242 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T03:17:43,243 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:43,243 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T03:17:43,243 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731899863243"}]},"ts":"9223372036854775807"} 2024-11-18T03:17:43,244 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T03:17:43,244 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 66ab176d63aa2fde549e454fab5778b4, NAME => 'TestAcidGuarantees,,1731899835236.66ab176d63aa2fde549e454fab5778b4.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T03:17:43,244 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T03:17:43,244 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731899863244"}]},"ts":"9223372036854775807"} 2024-11-18T03:17:43,245 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T03:17:43,246 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T03:17:43,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-18T03:17:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-18T03:17:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T03:17:43,316 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T03:17:43,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=253 (was 252) - Thread LEAK? -, OpenFileDescriptor=463 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1286 (was 1274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1684 (was 1781) 2024-11-18T03:17:43,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T03:17:43,326 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:17:43,326 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:17:43,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:43,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:43,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:17:43,326 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T03:17:43,326 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-18T03:17:43,327 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fd862c3a04f9,45051,1731899683325 2024-11-18T03:17:43,371 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:17:43,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:17:43,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:17:43,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T03:17:43,372 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:17:43,373 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:17:43,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:17:43,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:17:43,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:17:43,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:43,374 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fd862c3a04f9,38549,1731899684479' ***** 2024-11-18T03:17:43,374 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T03:17:43,375 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T03:17:43,375 INFO [RS:0;fd862c3a04f9:38549 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T03:17:43,375 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T03:17:43,375 INFO [RS:0;fd862c3a04f9:38549 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T03:17:43,375 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(959): stopping server fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:43,375 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:17:43,375 INFO [RS:0;fd862c3a04f9:38549 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fd862c3a04f9:38549. 2024-11-18T03:17:43,376 DEBUG [RS:0;fd862c3a04f9:38549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:17:43,376 DEBUG [RS:0;fd862c3a04f9:38549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:43,376 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T03:17:43,376 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T03:17:43,377 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T03:17:43,377 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T03:17:43,378 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T03:17:43,378 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T03:17:43,378 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:17:43,378 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:17:43,378 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:17:43,378 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:43,378 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:17:43,378 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:17:43,378 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-18T03:17:43,399 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/info/377a233cd1d1453091e45652488e78f5 is 95, key is TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3./info:/1731899718318/DeleteFamily/seqid=0 2024-11-18T03:17:43,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742521_1697 (size=6514) 2024-11-18T03:17:43,458 INFO [regionserver/fd862c3a04f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:17:43,574 INFO [regionserver/fd862c3a04f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T03:17:43,574 INFO [regionserver/fd862c3a04f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T03:17:43,578 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:43,778 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:43,805 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/info/377a233cd1d1453091e45652488e78f5 2024-11-18T03:17:43,825 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/ns/6f5a1cba1fdd4c7e8670e378357ef822 is 93, key is TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3./ns:/1731899718260/DeleteFamily/seqid=0 2024-11-18T03:17:43,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742522_1698 (size=5906) 2024-11-18T03:17:43,870 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T03:17:43,979 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:44,179 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:44,228 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/ns/6f5a1cba1fdd4c7e8670e378357ef822 2024-11-18T03:17:44,244 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/rep_barrier/78e6385100e5412daccc75ed2691cd65 is 102, key is TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3./rep_barrier:/1731899718260/DeleteFamily/seqid=0 2024-11-18T03:17:44,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742523_1699 (size=6025) 2024-11-18T03:17:44,379 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:44,579 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:44,647 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/rep_barrier/78e6385100e5412daccc75ed2691cd65 2024-11-18T03:17:44,670 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/table/18f42a39c9004ce59873f8fff536a2c8 is 96, key is TestAcidGuarantees,,1731899689091.29141263847cbdef9bad2179de1271a3./table:/1731899718260/DeleteFamily/seqid=0 2024-11-18T03:17:44,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742524_1700 (size=5936) 2024-11-18T03:17:44,779 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:44,980 DEBUG [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:17:45,074 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/table/18f42a39c9004ce59873f8fff536a2c8 2024-11-18T03:17:45,081 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/info/377a233cd1d1453091e45652488e78f5 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/info/377a233cd1d1453091e45652488e78f5 2024-11-18T03:17:45,088 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/info/377a233cd1d1453091e45652488e78f5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-18T03:17:45,089 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/ns/6f5a1cba1fdd4c7e8670e378357ef822 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/ns/6f5a1cba1fdd4c7e8670e378357ef822 2024-11-18T03:17:45,092 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/ns/6f5a1cba1fdd4c7e8670e378357ef822, entries=8, sequenceid=90, filesize=5.8 K 2024-11-18T03:17:45,093 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/rep_barrier/78e6385100e5412daccc75ed2691cd65 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/rep_barrier/78e6385100e5412daccc75ed2691cd65 2024-11-18T03:17:45,096 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/rep_barrier/78e6385100e5412daccc75ed2691cd65, entries=6, sequenceid=90, filesize=5.9 K 2024-11-18T03:17:45,096 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/.tmp/table/18f42a39c9004ce59873f8fff536a2c8 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/table/18f42a39c9004ce59873f8fff536a2c8 2024-11-18T03:17:45,099 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/table/18f42a39c9004ce59873f8fff536a2c8, entries=7, sequenceid=90, filesize=5.8 K 2024-11-18T03:17:45,100 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1722ms, sequenceid=90, compaction requested=false 2024-11-18T03:17:45,103 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-18T03:17:45,104 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T03:17:45,104 INFO [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:17:45,104 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731899863377Running coprocessor pre-close hooks at 1731899863378 (+1 ms)Disabling compacts and flushes for region at 1731899863378Disabling writes for close at 1731899863378Obtaining lock to block concurrent updates at 1731899863378Preparing flush snapshotting stores in 1588230740 at 1731899863378Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731899863379 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731899863379Flushing 1588230740/info: creating writer at 1731899863379Flushing 1588230740/info: appending metadata at 1731899863399 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731899863399Flushing 1588230740/ns: creating writer at 1731899863810 (+411 ms)Flushing 1588230740/ns: appending metadata at 1731899863825 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731899863825Flushing 1588230740/rep_barrier: creating writer at 1731899864231 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731899864244 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731899864244Flushing 1588230740/table: creating writer at 1731899864652 (+408 ms)Flushing 1588230740/table: appending metadata at 1731899864669 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731899864670 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63ae94e3: reopening flushed file at 1731899865080 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e30653: reopening flushed file at 1731899865088 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25022efb: reopening flushed file at 1731899865093 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6200bad4: reopening flushed file at 1731899865096 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1722ms, sequenceid=90, compaction requested=false at 1731899865100 (+4 ms)Writing region close event to WAL at 1731899865101 (+1 ms)Running coprocessor post-close hooks at 1731899865104 (+3 ms)Closed at 1731899865104 2024-11-18T03:17:45,104 DEBUG [RS_CLOSE_META-regionserver/fd862c3a04f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T03:17:45,180 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(976): stopping server fd862c3a04f9,38549,1731899684479; all regions closed. 2024-11-18T03:17:45,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741834_1010 (size=25344) 2024-11-18T03:17:45,186 DEBUG [RS:0;fd862c3a04f9:38549 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/oldWALs 2024-11-18T03:17:45,186 INFO [RS:0;fd862c3a04f9:38549 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fd862c3a04f9%2C38549%2C1731899684479.meta:.meta(num 1731899688628) 2024-11-18T03:17:45,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741832_1008 (size=17450555) 2024-11-18T03:17:45,192 DEBUG [RS:0;fd862c3a04f9:38549 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/oldWALs 2024-11-18T03:17:45,192 INFO [RS:0;fd862c3a04f9:38549 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fd862c3a04f9%2C38549%2C1731899684479:(num 1731899687673) 2024-11-18T03:17:45,192 DEBUG [RS:0;fd862c3a04f9:38549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:17:45,192 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:17:45,192 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:17:45,192 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.ChoreService(370): Chore service for: regionserver/fd862c3a04f9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T03:17:45,193 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:17:45,193 INFO [regionserver/fd862c3a04f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:17:45,194 INFO [RS:0;fd862c3a04f9:38549 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38549 2024-11-18T03:17:45,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:17:45,222 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fd862c3a04f9,38549,1731899684479 2024-11-18T03:17:45,222 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:17:45,223 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fd862c3a04f9,38549,1731899684479] 2024-11-18T03:17:45,411 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:17:45,412 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38549-0x1014befe9de0001, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:17:45,412 INFO [RS:0;fd862c3a04f9:38549 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:17:45,412 INFO [RS:0;fd862c3a04f9:38549 {}] regionserver.HRegionServer(1031): Exiting; stopping=fd862c3a04f9,38549,1731899684479; zookeeper connection closed. 2024-11-18T03:17:45,413 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@380baf6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@380baf6 2024-11-18T03:17:45,413 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T03:17:45,413 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fd862c3a04f9,38549,1731899684479 already deleted, retry=false 2024-11-18T03:17:45,414 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fd862c3a04f9,38549,1731899684479 expired; onlineServers=0 2024-11-18T03:17:45,414 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fd862c3a04f9,45051,1731899683325' ***** 2024-11-18T03:17:45,414 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T03:17:45,414 INFO [M:0;fd862c3a04f9:45051 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:17:45,414 INFO [M:0;fd862c3a04f9:45051 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:17:45,414 DEBUG [M:0;fd862c3a04f9:45051 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T03:17:45,414 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T03:17:45,414 DEBUG [M:0;fd862c3a04f9:45051 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T03:17:45,414 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster-HFileCleaner.large.0-1731899687298 {}] cleaner.HFileCleaner(306): Exit Thread[master/fd862c3a04f9:0:becomeActiveMaster-HFileCleaner.large.0-1731899687298,5,FailOnTimeoutGroup] 2024-11-18T03:17:45,414 DEBUG [master/fd862c3a04f9:0:becomeActiveMaster-HFileCleaner.small.0-1731899687299 {}] cleaner.HFileCleaner(306): Exit Thread[master/fd862c3a04f9:0:becomeActiveMaster-HFileCleaner.small.0-1731899687299,5,FailOnTimeoutGroup] 2024-11-18T03:17:45,415 INFO [M:0;fd862c3a04f9:45051 {}] hbase.ChoreService(370): Chore service for: master/fd862c3a04f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T03:17:45,415 INFO [M:0;fd862c3a04f9:45051 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:17:45,415 DEBUG [M:0;fd862c3a04f9:45051 {}] master.HMaster(1795): Stopping service threads 2024-11-18T03:17:45,415 INFO [M:0;fd862c3a04f9:45051 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T03:17:45,415 INFO [M:0;fd862c3a04f9:45051 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T03:17:45,415 ERROR [M:0;fd862c3a04f9:45051 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-18T03:17:45,416 INFO [M:0;fd862c3a04f9:45051 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T03:17:45,416 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T03:17:45,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T03:17:45,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:17:45,510 DEBUG [M:0;fd862c3a04f9:45051 {}] zookeeper.ZKUtil(347): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T03:17:45,510 WARN [M:0;fd862c3a04f9:45051 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T03:17:45,512 INFO [M:0;fd862c3a04f9:45051 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/.lastflushedseqids 2024-11-18T03:17:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742525_1701 (size=72) 2024-11-18T03:17:45,526 INFO [M:0;fd862c3a04f9:45051 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T03:17:45,526 INFO [M:0;fd862c3a04f9:45051 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T03:17:45,526 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:17:45,526 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:17:45,526 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:17:45,526 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:17:45,526 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:17:45,526 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=772.64 KB heapSize=945.80 KB 2024-11-18T03:17:45,540 DEBUG [M:0;fd862c3a04f9:45051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dba6bfb1de6a48b7b1889f1a3e08df14 is 82, key is hbase:meta,,1/info:regioninfo/1731899688710/Put/seqid=0 2024-11-18T03:17:45,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742526_1702 (size=5672) 2024-11-18T03:17:45,944 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2135 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dba6bfb1de6a48b7b1889f1a3e08df14 2024-11-18T03:17:45,975 DEBUG [M:0;fd862c3a04f9:45051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d91710ad191a42d482a76519e0fa088b is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731899838339/Put/seqid=0 2024-11-18T03:17:45,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742527_1703 (size=45362) 2024-11-18T03:17:46,378 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=772.08 KB at sequenceid=2135 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d91710ad191a42d482a76519e0fa088b 2024-11-18T03:17:46,382 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d91710ad191a42d482a76519e0fa088b 2024-11-18T03:17:46,398 DEBUG [M:0;fd862c3a04f9:45051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e135d880ea04fb8837000795ce660f9 is 69, key is fd862c3a04f9,38549,1731899684479/rs:state/1731899687419/Put/seqid=0 2024-11-18T03:17:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073742528_1704 (size=5156) 2024-11-18T03:17:46,802 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2135 (bloomFilter=true), to=hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e135d880ea04fb8837000795ce660f9 2024-11-18T03:17:46,806 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dba6bfb1de6a48b7b1889f1a3e08df14 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dba6bfb1de6a48b7b1889f1a3e08df14 2024-11-18T03:17:46,809 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dba6bfb1de6a48b7b1889f1a3e08df14, entries=8, sequenceid=2135, filesize=5.5 K 2024-11-18T03:17:46,809 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d91710ad191a42d482a76519e0fa088b as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d91710ad191a42d482a76519e0fa088b 2024-11-18T03:17:46,811 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d91710ad191a42d482a76519e0fa088b 2024-11-18T03:17:46,812 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d91710ad191a42d482a76519e0fa088b, entries=170, sequenceid=2135, filesize=44.3 K 2024-11-18T03:17:46,812 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e135d880ea04fb8837000795ce660f9 as hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2e135d880ea04fb8837000795ce660f9 2024-11-18T03:17:46,814 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35537/user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2e135d880ea04fb8837000795ce660f9, entries=1, sequenceid=2135, filesize=5.0 K 2024-11-18T03:17:46,815 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(3140): Finished flush of dataSize ~772.64 KB/791183, heapSize ~945.51 KB/968200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1289ms, sequenceid=2135, compaction requested=false 2024-11-18T03:17:46,816 INFO [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:17:46,816 DEBUG [M:0;fd862c3a04f9:45051 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731899865526Disabling compacts and flushes for region at 1731899865526Disabling writes for close at 1731899865526Obtaining lock to block concurrent updates at 1731899865526Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731899865526Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=791183, getHeapSize=968440, getOffHeapSize=0, getCellsCount=2330 at 1731899865526Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731899865527 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731899865527Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731899865540 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731899865540Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731899865951 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731899865974 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731899865974Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731899866382 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731899866398 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731899866398Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b866dd: reopening flushed file at 1731899866805 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16bfa623: reopening flushed file at 1731899866809 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@382447cd: reopening flushed file at 1731899866812 (+3 ms)Finished flush of dataSize ~772.64 KB/791183, heapSize ~945.51 KB/968200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1289ms, sequenceid=2135, compaction requested=false at 1731899866815 (+3 ms)Writing region close event to WAL at 1731899866816 (+1 ms)Closed at 1731899866816 2024-11-18T03:17:46,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44343 is added to blk_1073741830_1006 (size=931157) 2024-11-18T03:17:46,820 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/361778a0-a79d-f45c-d673-f575babc1fe1/MasterData/WALs/fd862c3a04f9,45051,1731899683325/fd862c3a04f9%2C45051%2C1731899683325.1731899686639 not finished, retry = 0 2024-11-18T03:17:46,922 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:17:46,922 INFO [M:0;fd862c3a04f9:45051 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T03:17:46,922 INFO [M:0;fd862c3a04f9:45051 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45051 2024-11-18T03:17:46,923 INFO [M:0;fd862c3a04f9:45051 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:17:47,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:17:47,153 INFO [M:0;fd862c3a04f9:45051 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:17:47,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1014befe9de0000, quorum=127.0.0.1:56749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:17:47,160 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70061d7e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T03:17:47,164 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T03:17:47,164 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T03:17:47,164 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T03:17:47,164 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/hadoop.log.dir/,STOPPED} 2024-11-18T03:17:47,167 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T03:17:47,167 WARN [BP-39717385-172.17.0.3-1731899678439 heartbeating to localhost/127.0.0.1:35537 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T03:17:47,167 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T03:17:47,167 WARN [BP-39717385-172.17.0.3-1731899678439 heartbeating to localhost/127.0.0.1:35537 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-39717385-172.17.0.3-1731899678439 (Datanode Uuid 58cfbc07-69ae-4c4e-8282-2160a7242b4a) service to localhost/127.0.0.1:35537 2024-11-18T03:17:47,169 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/data/data1/current/BP-39717385-172.17.0.3-1731899678439 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T03:17:47,169 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/cluster_47f149f6-545d-d128-d3b1-866ea683b1c7/data/data2/current/BP-39717385-172.17.0.3-1731899678439 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T03:17:47,170 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T03:17:47,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f7463d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T03:17:47,177 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T03:17:47,177 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T03:17:47,178 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T03:17:47,178 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/75aeddb4-b90d-49e3-2dc9-076fbd3dafa5/hadoop.log.dir/,STOPPED} 2024-11-18T03:17:47,193 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T03:17:47,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down